alexion/dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(vendor): import pi 1.3.0

Browse Source 
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
alexion
2026-10-15 17:58:48 -04:00
parent 719370c8f7
commit 6081e3e325
1743 changed files with 205460 additions and 131705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

184
third_party/pi/packages/durable/src/tools/bash.ts vendored Normal file
View File

@@ -0,0 +1,184 @@
import type { Context } from "@earendil-works/chord";
import { type Static, Type } from "typebox";
import { defineTool } from "../harness/define.ts";
import type { ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts";
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES } from "../truncate.ts";
import { requireEnv } from "./env.ts";
const MAX_TIMEOUT_SECONDS = 2_147_483_647 / 1000;
const bashSchema = Type.Object({
command: Type.String({ description: "Bash command to execute" }),
timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
});
const powershellSchema = Type.Object({
command: Type.String({ description: "PowerShell command to execute" }),
timeout: Type.Optional(Type.Number({ description: "Timeout in seconds (optional, no default timeout)" })),
});
/** What a program that calls `bash` or `powershell` receives. */
const shellStructuredOutputSchema = Type.Object({
output: Type.String({ description: "Combined stdout and stderr: the retained tail, as the model sees it" }),
truncated: Type.Boolean({ description: "Whether earlier output was dropped" }),
fullOutputPath: Type.Optional(Type.String({ description: "File with the complete output, when it was spilled" })),
exitCode: Type.Number(),
});
export type BashToolInput = Static<typeof bashSchema>;
export type PowerShellToolInput = Static<typeof powershellSchema>;
export type ShellStructuredOutput = Static<typeof shellStructuredOutputSchema>;
/** A command about to run, which `prepare` may change: the script, its working directory and environment. */
export interface BashExecution {
command: string;
cwd: string;
env: Record<string, string>;
inheritEnv: boolean;
}
export type BashPrepare = (execution: BashExecution, api: ToolExecutionApi, context: Context) => void | Promise<void>;
export interface BashToolOptions {
commandPrefix?: string;
prepare?: BashPrepare;
}
export interface PowerShellToolOptions {
/** Lines run before each command. */
commandPrefix?: string;
prepare?: BashPrepare;
/** PowerShell programs to try in order; default `pwsh`, then `powershell`. A program that cannot start is skipped. */
programs?: readonly string[];
}
function validateTimeout(timeout: number | undefined): void {
if (timeout === undefined) return;
if (!Number.isFinite(timeout) || timeout <= 0) {
throw new Error("Invalid timeout: must be a finite number of seconds");
}
if (timeout > MAX_TIMEOUT_SECONDS) {
throw new Error(`Invalid timeout: maximum is ${MAX_TIMEOUT_SECONDS} seconds`);
}
}
/** The execution of one call: the command with its prefix, in the environment's working directory, then `prepare`. */
async function prepareExecution(
command: string,
options: { commandPrefix?: string; prepare?: BashPrepare } | undefined,
api: ToolExecutionApi,
context: Context,
): Promise<BashExecution> {
const execution: BashExecution = {
command: options?.commandPrefix ? `${options.commandPrefix}\n${command}` : command,
cwd: requireEnv(api).cwd,
env: {},
inheritEnv: true,
};
await options?.prepare?.(execution, api, context);
return execution;
}
/**
* Run each command form in turn until one starts, streaming output to `api.output()` within the retained window, and
* turn the result into the tool's result: a spill diagnostic, an error result for a nonzero exit, and a thrown error for
* a failure such as a timeout. The structured output repeats the retained output with the exit code.
*/
async function runCommand(
commands: readonly (string | readonly string[])[],
execution: BashExecution,
timeout: number | undefined,
api: ToolExecutionApi,
context: Context,
): Promise<ToolExecutionResult> {
const env = requireEnv(api);
let result: Awaited<ReturnType<typeof env.exec>> | undefined;
for (const command of commands) {
result = await env.exec(
command,
{
cwd: execution.cwd,
env: execution.env,
inheritEnv: execution.inheritEnv,
...(timeout === undefined ? {} : { timeout }),
onOutput: (text, _context, info) => api.output(text, info.skipped),
spill: { afterBytes: DEFAULT_MAX_BYTES, afterLines: DEFAULT_MAX_LINES },
// An environment may then omit output outside the retained tail and report the omission.
...(api.outputWindow === undefined ? {} : { window: api.outputWindow }),
},
context,
);
// A program that could not start produced no output; the next form may.
if (result.ok || result.error.code !== "spawn_error") break;
}
if (result === undefined) throw new Error("No command to run");
const spillPath = result.ok ? result.value.spillPath : result.error.spillPath;
if (spillPath !== undefined) {
api.diagnostic({ severity: "info", code: "full_output", message: `Full output: ${spillPath}` });
}
if (!result.ok) {
if (result.error.code === "aborted" && context.abortSignal?.aborted) throw result.error;
if (result.error.code === "timeout") throw new Error(`Command timed out after ${timeout} seconds`);
if (result.error.code === "aborted") throw new Error("Command aborted");
throw result.error;
}
const exitCode = result.value.exitCode;
const retained = api.retainedOutput();
const structuredOutput: ShellStructuredOutput = {
output: retained.text,
truncated: retained.truncated,
...(spillPath === undefined ? {} : { fullOutputPath: spillPath }),
exitCode,
};
if (exitCode === 0) return { structuredOutput };
const exited = { severity: "error", code: "exit_code", message: `Command exited with code ${exitCode}` } as const;
return { structuredOutput, isError: true, diagnostics: [exited] };
}
/**
* Runs a command through the environment's shell. Its output streams to `api.output()`, where the Harness keeps the
* tail within the default limits; the result content is that retained output. The retained window goes to the
* environment, which may omit output outside it and report how much it omitted, so dropped counts stay exact. Output
* beyond the limits is spilled to a file whose path is reported as a diagnostic. A nonzero exit is an error result with
* the exit code; a timeout throws, which makes an error result that still carries the output and diagnostics.
*/
export function createBashTool(options?: BashToolOptions): ToolRegistration<typeof bashSchema> {
return defineTool({
name: "bash",
description: `Execute a bash command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
parameters: bashSchema,
structuredOutputSchema: shellStructuredOutputSchema,
outputLimits: { retain: "tail" },
async execute(args, api, context) {
validateTimeout(args.timeout);
const execution = await prepareExecution(args.command, options, api, context);
return runCommand([execution.command], execution, args.timeout, api, context);
},
});
}
/** Output in UTF-8 whatever the console's code page, as the coding agent's `powershell` tool does. */
const UTF8_OUTPUT = "try { [Console]::OutputEncoding=[System.Text.Encoding]::UTF8 } catch {}";
const POWERSHELL_ARGS = ["-NoProfile", "-NonInteractive", "-ExecutionPolicy", "Bypass", "-Command"] as const;
/**
* Runs a PowerShell command, like `bash` but through PowerShell instead of the environment's shell: `pwsh` (PowerShell
* 7), else Windows PowerShell, started directly with the command as an argument, so no other shell parses it.
*/
export function createPowerShellTool(options?: PowerShellToolOptions): ToolRegistration<typeof powershellSchema> {
const programs = options?.programs ?? ["pwsh", "powershell"];
return defineTool({
name: "powershell",
description: `Execute a PowerShell command in the current working directory. Returns combined stdout and stderr. Output is truncated to last ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). If truncated, full output is saved to a temp file. Optionally provide a timeout in seconds.`,
parameters: powershellSchema,
structuredOutputSchema: shellStructuredOutputSchema,
outputLimits: { retain: "tail" },
async execute(args, api, context) {
validateTimeout(args.timeout);
const execution = await prepareExecution(args.command, options, api, context);
const script = `${UTF8_OUTPUT}\n${execution.command}`;
const commands = programs.map((program) => [program, ...POWERSHELL_ARGS, script]);
return runCommand(commands, execution, args.timeout, api, context);
},
});
}

500
third_party/pi/packages/durable/src/tools/edit-diff.ts vendored Normal file
View File

@@ -0,0 +1,500 @@
/**
* Shared diff computation utilities for the edit and similar tools.
*/
import * as Diff from "diff";
export function detectLineEnding(content: string): "\r\n" | "\n" {
const crlfIdx = content.indexOf("\r\n");
const lfIdx = content.indexOf("\n");
if (lfIdx === -1) return "\n";
if (crlfIdx === -1) return "\n";
return crlfIdx < lfIdx ? "\r\n" : "\n";
}
export function normalizeToLF(text: string): string {
return text.replace(/\r\n/g, "\n").replace(/\r/g, "\n");
}
export function restoreLineEndings(text: string, ending: "\r\n" | "\n"): string {
return ending === "\r\n" ? text.replace(/\n/g, "\r\n") : text;
}
/**
* Normalize text for fuzzy matching. Applies progressive transformations:
* - Strip trailing whitespace from each line
* - Normalize smart quotes to ASCII equivalents
* - Normalize Unicode dashes/hyphens to ASCII hyphen
* - Normalize special Unicode spaces to regular space
*/
export function normalizeForFuzzyMatch(text: string): string {
return (
text
.normalize("NFKC")
// Strip trailing whitespace per line
.split("\n")
.map((line) => line.trimEnd())
.join("\n")
// Smart single quotes → '
.replace(/[\u2018\u2019\u201A\u201B]/g, "'")
// Smart double quotes → "
.replace(/[\u201C\u201D\u201E\u201F]/g, '"')
// Various dashes/hyphens → -
// U+2010 hyphen, U+2011 non-breaking hyphen, U+2012 figure dash,
// U+2013 en-dash, U+2014 em-dash, U+2015 horizontal bar, U+2212 minus
.replace(/[\u2010\u2011\u2012\u2013\u2014\u2015\u2212]/g, "-")
// Special spaces → regular space
// U+00A0 NBSP, U+2002-U+200A various spaces, U+202F narrow NBSP,
// U+205F medium math space, U+3000 ideographic space
.replace(/[\u00A0\u2002-\u200A\u202F\u205F\u3000]/g, " ")
);
}
function splitLinesWithEndings(content: string): string[] {
return content.match(/[^\n]*\n|[^\n]+/g) ?? [];
}
interface LineSpan {
start: number;
end: number;
}
interface MatchedEdit {
editIndex: number;
matchIndex: number;
matchLength: number;
newText: string;
}
type TextReplacement = Pick<MatchedEdit, "matchIndex" | "matchLength" | "newText">;
function getLineSpans(content: string): LineSpan[] {
let offset = 0;
return splitLinesWithEndings(content).map((line) => {
const span = { start: offset, end: offset + line.length };
offset = span.end;
return span;
});
}
function getReplacementLineRange(lines: LineSpan[], replacement: TextReplacement) {
const replacementStart = replacement.matchIndex;
const replacementEnd = replacement.matchIndex + replacement.matchLength;
let startLine = -1;
for (let i = 0; i < lines.length; i++) {
const line = lines[i];
if (replacementStart >= line.start && replacementStart < line.end) {
startLine = i;
break;
}
}
if (startLine === -1) {
throw new Error("Replacement range is outside the base content.");
}
let endLine = startLine;
while (endLine < lines.length && lines[endLine].end < replacementEnd) {
endLine++;
}
if (endLine >= lines.length) {
throw new Error("Replacement range is outside the base content.");
}
return { startLine, endLine: endLine + 1 };
}
function applyReplacements(content: string, replacements: TextReplacement[], offset = 0): string {
let result = content;
for (let i = replacements.length - 1; i >= 0; i--) {
const replacement = replacements[i];
const matchIndex = replacement.matchIndex - offset;
result =
result.substring(0, matchIndex) + replacement.newText + result.substring(matchIndex + replacement.matchLength);
}
return result;
}
/**
* Apply replacements matched against `baseContent` to `originalContent` while
* preserving unchanged line blocks from the original.
*
* This is useful when `baseContent` is a normalized view of the original. Each
* replacement is widened to the lines it actually touches, those touched lines
* are rewritten from the normalized base, and all other lines are copied back
* from `originalContent`. The actual replacement ranges drive preservation so
* duplicate normalized lines cannot be aligned to the wrong occurrence.
*/
export function applyReplacementsPreservingUnchangedLines(
originalContent: string,
baseContent: string,
replacements: TextReplacement[],
): string {
const originalLines = splitLinesWithEndings(originalContent);
const baseLines = getLineSpans(baseContent);
if (originalLines.length !== baseLines.length) {
throw new Error("Cannot preserve unchanged lines because the base content has a different line count.");
}
const groups: Array<{ startLine: number; endLine: number; replacements: TextReplacement[] }> = [];
const sortedReplacements = [...replacements].sort((a, b) => a.matchIndex - b.matchIndex);
for (const replacement of sortedReplacements) {
const range = getReplacementLineRange(baseLines, replacement);
const current = groups[groups.length - 1];
if (current && range.startLine < current.endLine) {
current.endLine = Math.max(current.endLine, range.endLine);
current.replacements.push(replacement);
continue;
}
groups.push({ ...range, replacements: [replacement] });
}
let originalLineIndex = 0;
let result = "";
for (const group of groups) {
result += originalLines.slice(originalLineIndex, group.startLine).join("");
const groupStartOffset = baseLines[group.startLine].start;
const groupEndOffset = baseLines[group.endLine - 1].end;
result += applyReplacements(
baseContent.slice(groupStartOffset, groupEndOffset),
group.replacements,
groupStartOffset,
);
originalLineIndex = group.endLine;
}
result += originalLines.slice(originalLineIndex).join("");
return result;
}
export interface FuzzyMatchResult {
/** Whether a match was found */
found: boolean;
/** The index where the match starts (in the content that should be used for replacement) */
index: number;
/** Length of the matched text */
matchLength: number;
/** Whether fuzzy matching was used (false = exact match) */
usedFuzzyMatch: boolean;
/**
* The content to use for replacement operations.
* When exact match: original content. When fuzzy match: normalized content.
*/
contentForReplacement: string;
}
export interface Edit {
oldText: string;
newText: string;
}
export interface AppliedEditsResult {
baseContent: string;
newContent: string;
}
/**
* Find oldText in content, trying exact match first, then fuzzy match.
* When fuzzy matching is used, the returned contentForReplacement is the
* fuzzy-normalized version of the content (trailing whitespace stripped,
* Unicode quotes/dashes normalized to ASCII).
*/
export function fuzzyFindText(content: string, oldText: string): FuzzyMatchResult {
// Try exact match first
const exactIndex = content.indexOf(oldText);
if (exactIndex !== -1) {
return {
found: true,
index: exactIndex,
matchLength: oldText.length,
usedFuzzyMatch: false,
contentForReplacement: content,
};
}
// Try fuzzy match - work entirely in normalized space
const fuzzyContent = normalizeForFuzzyMatch(content);
const fuzzyOldText = normalizeForFuzzyMatch(oldText);
const fuzzyIndex = fuzzyContent.indexOf(fuzzyOldText);
if (fuzzyIndex === -1) {
return {
found: false,
index: -1,
matchLength: 0,
usedFuzzyMatch: false,
contentForReplacement: content,
};
}
// When fuzzy matching, return offsets in normalized space. Callers can use
// the normalized content to compute replacements, then decide how much of
// that normalized output should be written back.
return {
found: true,
index: fuzzyIndex,
matchLength: fuzzyOldText.length,
usedFuzzyMatch: true,
contentForReplacement: fuzzyContent,
};
}
/** Strip UTF-8 BOM if present, return both the BOM (if any) and the text without it */
export function stripBom(content: string): { bom: string; text: string } {
return content.startsWith("\uFEFF") ? { bom: "\uFEFF", text: content.slice(1) } : { bom: "", text: content };
}
function countOccurrences(content: string, oldText: string): number {
const fuzzyContent = normalizeForFuzzyMatch(content);
const fuzzyOldText = normalizeForFuzzyMatch(oldText);
return fuzzyContent.split(fuzzyOldText).length - 1;
}
function getNotFoundError(path: string, editIndex: number, totalEdits: number): Error {
if (totalEdits === 1) {
return new Error(
`Could not find the exact text in ${path}. The old text must match exactly including all whitespace and newlines.`,
);
}
return new Error(
`Could not find edits[${editIndex}] in ${path}. The oldText must match exactly including all whitespace and newlines.`,
);
}
function getDuplicateError(path: string, editIndex: number, totalEdits: number, occurrences: number): Error {
if (totalEdits === 1) {
return new Error(
`Found ${occurrences} occurrences of the text in ${path}. The text must be unique. Please provide more context to make it unique.`,
);
}
return new Error(
`Found ${occurrences} occurrences of edits[${editIndex}] in ${path}. Each oldText must be unique. Please provide more context to make it unique.`,
);
}
function getEmptyOldTextError(path: string, editIndex: number, totalEdits: number): Error {
if (totalEdits === 1) {
return new Error(`oldText must not be empty in ${path}.`);
}
return new Error(`edits[${editIndex}].oldText must not be empty in ${path}.`);
}
function getNoChangeError(path: string, totalEdits: number): Error {
if (totalEdits === 1) {
return new Error(
`No changes made to ${path}. The replacement produced identical content. This might indicate an issue with special characters or the text not existing as expected.`,
);
}
return new Error(`No changes made to ${path}. The replacements produced identical content.`);
}
/**
* Apply one or more exact-text replacements to LF-normalized content.
*
* All edits are matched against the same original content. Replacements are
* then applied in reverse order so offsets remain stable. If any edit needs
* fuzzy matching, the operation runs in fuzzy-normalized content space and then
* overlays those line-level changes onto the original content so unchanged line
* blocks keep their original bytes.
*/
export function applyEditsToNormalizedContent(
normalizedContent: string,
edits: Edit[],
path: string,
): AppliedEditsResult {
const normalizedEdits = edits.map((edit) => ({
oldText: normalizeToLF(edit.oldText),
newText: normalizeToLF(edit.newText),
}));
for (let i = 0; i < normalizedEdits.length; i++) {
if (normalizedEdits[i].oldText.length === 0) {
throw getEmptyOldTextError(path, i, normalizedEdits.length);
}
}
const initialMatches = normalizedEdits.map((edit) => fuzzyFindText(normalizedContent, edit.oldText));
const usedFuzzyMatch = initialMatches.some((match) => match.usedFuzzyMatch);
const replacementBaseContent = usedFuzzyMatch ? normalizeForFuzzyMatch(normalizedContent) : normalizedContent;
const matchedEdits: MatchedEdit[] = [];
for (let i = 0; i < normalizedEdits.length; i++) {
const edit = normalizedEdits[i];
const matchResult = fuzzyFindText(replacementBaseContent, edit.oldText);
if (!matchResult.found) {
throw getNotFoundError(path, i, normalizedEdits.length);
}
const occurrences = countOccurrences(replacementBaseContent, edit.oldText);
if (occurrences > 1) {
throw getDuplicateError(path, i, normalizedEdits.length, occurrences);
}
matchedEdits.push({
editIndex: i,
matchIndex: matchResult.index,
matchLength: matchResult.matchLength,
newText: edit.newText,
});
}
matchedEdits.sort((a, b) => a.matchIndex - b.matchIndex);
for (let i = 1; i < matchedEdits.length; i++) {
const previous = matchedEdits[i - 1];
const current = matchedEdits[i];
if (previous.matchIndex + previous.matchLength > current.matchIndex) {
throw new Error(
`edits[${previous.editIndex}] and edits[${current.editIndex}] overlap in ${path}. Merge them into one edit or target disjoint regions.`,
);
}
}
const baseContent = normalizedContent;
const newContent = usedFuzzyMatch
? applyReplacementsPreservingUnchangedLines(normalizedContent, replacementBaseContent, matchedEdits)
: applyReplacements(replacementBaseContent, matchedEdits);
if (baseContent === newContent) {
throw getNoChangeError(path, normalizedEdits.length);
}
return { baseContent, newContent };
}
/** Generate a standard unified patch. */
export function generateUnifiedPatch(path: string, oldContent: string, newContent: string, contextLines = 4): string {
return Diff.createTwoFilesPatch(path, path, oldContent, newContent, undefined, undefined, {
context: contextLines,
headerOptions: Diff.FILE_HEADERS_ONLY,
});
}
/**
* Generate a display-oriented diff string with line numbers and context.
* Returns both the diff string and the first changed line number (in the new file).
*/
export function generateDiffString(
oldContent: string,
newContent: string,
contextLines = 4,
): { diff: string; firstChangedLine: number | undefined } {
const parts = Diff.diffLines(oldContent, newContent);
const output: string[] = [];
const oldLines = oldContent.split("\n");
const newLines = newContent.split("\n");
const maxLineNum = Math.max(oldLines.length, newLines.length);
const lineNumWidth = String(maxLineNum).length;
let oldLineNum = 1;
let newLineNum = 1;
let lastWasChange = false;
let firstChangedLine: number | undefined;
for (let i = 0; i < parts.length; i++) {
const part = parts[i];
const raw = part.value.split("\n");
if (raw[raw.length - 1] === "") {
raw.pop();
}
if (part.added || part.removed) {
// Capture the first changed line (in the new file)
if (firstChangedLine === undefined) {
firstChangedLine = newLineNum;
}
// Show the change
for (const line of raw) {
if (part.added) {
const lineNum = String(newLineNum).padStart(lineNumWidth, " ");
output.push(`+${lineNum} ${line}`);
newLineNum++;
} else {
// removed
const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
output.push(`-${lineNum} ${line}`);
oldLineNum++;
}
}
lastWasChange = true;
} else {
// Context lines - only show a few before/after changes
const nextPartIsChange = i < parts.length - 1 && (parts[i + 1].added || parts[i + 1].removed);
const hasLeadingChange = lastWasChange;
const hasTrailingChange = nextPartIsChange;
if (hasLeadingChange && hasTrailingChange) {
if (raw.length <= contextLines * 2) {
for (const line of raw) {
const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
output.push(` ${lineNum} ${line}`);
oldLineNum++;
newLineNum++;
}
} else {
const leadingLines = raw.slice(0, contextLines);
const trailingLines = raw.slice(raw.length - contextLines);
const skippedLines = raw.length - leadingLines.length - trailingLines.length;
for (const line of leadingLines) {
const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
output.push(` ${lineNum} ${line}`);
oldLineNum++;
newLineNum++;
}
output.push(` ${"".padStart(lineNumWidth, " ")} ...`);
oldLineNum += skippedLines;
newLineNum += skippedLines;
for (const line of trailingLines) {
const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
output.push(` ${lineNum} ${line}`);
oldLineNum++;
newLineNum++;
}
}
} else if (hasLeadingChange) {
const shownLines = raw.slice(0, contextLines);
const skippedLines = raw.length - shownLines.length;
for (const line of shownLines) {
const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
output.push(` ${lineNum} ${line}`);
oldLineNum++;
newLineNum++;
}
if (skippedLines > 0) {
output.push(` ${"".padStart(lineNumWidth, " ")} ...`);
oldLineNum += skippedLines;
newLineNum += skippedLines;
}
} else if (hasTrailingChange) {
const skippedLines = Math.max(0, raw.length - contextLines);
if (skippedLines > 0) {
output.push(` ${"".padStart(lineNumWidth, " ")} ...`);
oldLineNum += skippedLines;
newLineNum += skippedLines;
}
for (const line of raw.slice(skippedLines)) {
const lineNum = String(oldLineNum).padStart(lineNumWidth, " ");
output.push(` ${lineNum} ${line}`);
oldLineNum++;
newLineNum++;
}
} else {
// Skip these context lines entirely
oldLineNum += raw.length;
newLineNum += raw.length;
}
lastWasChange = false;
}
}
return { diff: output.join("\n"), firstChangedLine };
}

144
third_party/pi/packages/durable/src/tools/edit.ts vendored Normal file
View File

@@ -0,0 +1,144 @@
import { type Static, Type } from "typebox";
import type { FileError } from "../env/index.ts";
import { defineTool } from "../harness/define.ts";
import type { ToolRegistration } from "../harness/types.ts";
import {
applyEditsToNormalizedContent,
detectLineEnding,
type Edit,
generateDiffString,
generateUnifiedPatch,
normalizeToLF,
restoreLineEndings,
stripBom,
} from "./edit-diff.ts";
import { requireEnv } from "./env.ts";
import { withFileMutationQueue } from "./file-mutation-queue.ts";
import { resolveToolPath } from "./path-utils.ts";
const replaceEditSchema = Type.Object({
oldText: Type.String({
description:
"Exact text for one targeted replacement. It must be unique in the original file and must not overlap with any other edits[].oldText in the same call.",
}),
newText: Type.String({ description: "Replacement text for this targeted edit." }),
});
const editSchema = Type.Object({
path: Type.String({ description: "Path to the file to edit (relative or absolute)" }),
edits: Type.Array(replaceEditSchema, {
description:
"One or more targeted replacements. Each edit is matched against the original file, not incrementally. Do not include overlapping or nested edits. If two changes touch the same block or nearby lines, merge them into one edit instead.",
}),
});
export type EditToolInput = Static<typeof editSchema>;
type LegacyEditToolInput = EditToolInput & { oldText?: unknown; newText?: unknown };
type SingleEditInput = { oldText: string; newText: string };
function isSingleEditInput(value: unknown): value is SingleEditInput {
if (!value || typeof value !== "object" || Array.isArray(value)) return false;
const edit = value as Record<string, unknown>;
return typeof edit.oldText === "string" && typeof edit.newText === "string";
}
/**
* Repair shapes models commonly send: `edits` as a JSON string or as a single edit object, and a top-level
* `oldText`/`newText` pair. Works on a copy; the call's arguments stay unchanged.
*/
function prepareEditArguments(input: unknown): EditToolInput {
if (!input || typeof input !== "object" || Array.isArray(input)) return input as EditToolInput;
const args: Record<string, unknown> = { ...input };
if (typeof args.edits === "string") {
try {
const parsed: unknown = JSON.parse(args.edits);
if (Array.isArray(parsed)) {
args.edits = parsed;
} else if (isSingleEditInput(parsed)) {
args.edits = [parsed];
}
} catch {}
} else if (isSingleEditInput(args.edits)) {
args.edits = [args.edits];
}
const legacy = args as LegacyEditToolInput;
if (typeof legacy.oldText !== "string" || typeof legacy.newText !== "string") return legacy;
const edits = Array.isArray(legacy.edits) ? [...legacy.edits] : [];
edits.push({ oldText: legacy.oldText, newText: legacy.newText });
const { oldText: _oldText, newText: _newText, ...rest } = legacy;
return { ...rest, edits };
}
export type EditToolDetails = {
diff: string;
patch: string;
firstChangedLine?: number;
};
function validateEditInput(input: EditToolInput): { path: string; edits: Edit[] } {
if (!Array.isArray(input.edits) || input.edits.length === 0) {
throw new Error("Edit tool input is invalid. edits must contain at least one replacement.");
}
return { path: input.path, edits: input.edits };
}
function editAccessError(path: string, error: FileError): Error {
return new Error(`Could not edit file: ${path}. Error code: ${error.code}.`, { cause: error });
}
export function createEditTool(): ToolRegistration<typeof editSchema, EditToolDetails> {
return defineTool({
name: "edit",
description:
"Edit a single file using exact text replacement. Every edits[].oldText must match a unique, non-overlapping region of the original file. If two changes affect the same block or nearby lines, merge them into one edit instead of emitting overlapping edits. Do not include large unchanged regions just to connect distant changes.",
parameters: editSchema,
prepareArguments: prepareEditArguments,
async execute(args, api, context) {
const { path, edits } = validateEditInput(args);
const env = requireEnv(api);
const absolutePath = await resolveToolPath(env, path, context);
return withFileMutationQueue(
env,
absolutePath,
async () => {
if (context.abortSignal?.aborted) throw new Error("Operation aborted");
const info = await env.fileInfo(absolutePath, context);
if (!info.ok) throw editAccessError(path, info.error);
if (info.value.kind !== "file" && info.value.kind !== "symlink") {
throw new Error(`Could not edit file: ${path}. Path is not a file.`);
}
const readResult = await env.readTextFile(absolutePath, context);
if (!readResult.ok) throw editAccessError(path, readResult.error);
if (context.abortSignal?.aborted) throw new Error("Operation aborted");
const { bom, text: content } = stripBom(readResult.value);
const originalEnding = detectLineEnding(content);
const normalizedContent = normalizeToLF(content);
const { baseContent, newContent } = applyEditsToNormalizedContent(normalizedContent, edits, path);
if (context.abortSignal?.aborted) throw new Error("Operation aborted");
const finalContent = bom + restoreLineEndings(newContent, originalEnding);
const writeResult = await env.writeFile(absolutePath, finalContent, context);
if (!writeResult.ok) throw editAccessError(path, writeResult.error);
if (context.abortSignal?.aborted) throw new Error("Operation aborted");
const diffResult = generateDiffString(baseContent, newContent);
const details: EditToolDetails = {
diff: diffResult.diff,
patch: generateUnifiedPatch(path, baseContent, newContent),
...(diffResult.firstChangedLine === undefined
? {}
: { firstChangedLine: diffResult.firstChangedLine }),
};
return {
output: [{ type: "text", text: `Successfully replaced ${edits.length} block(s) in ${path}.` }],
details,
};
},
context,
);
},
});
}

8
third_party/pi/packages/durable/src/tools/env.ts vendored Normal file
View File

@@ -0,0 +1,8 @@
import type { ExecutionEnv } from "../env/index.ts";
import type { ToolExecutionApi } from "../harness/types.ts";
/** The call's execution environment; a tool without one fails with an ordinary error result. */
export function requireEnv(api: ToolExecutionApi): ExecutionEnv {
if (api.env === undefined) throw new Error("No execution environment is configured");
return api.env;
}

55
third_party/pi/packages/durable/src/tools/file-mutation-queue.ts vendored Normal file
View File

@@ -0,0 +1,55 @@
import type { Context } from "@earendil-works/chord";
import { type ExecutionEnv, getOrThrow } from "../env/index.ts";
/** Tail of the mutation chain of each file, keyed by file system id and canonical path. */
const queues = new Map<string, Promise<void>>();
async function mutationKey(env: ExecutionEnv, path: string, context: Context): Promise<string> {
const absolutePath = getOrThrow(await env.absolutePath(path, context));
return `${env.id}\0${await canonical(env, absolutePath, context)}`;
}
/**
* The canonical path; for a file that does not exist yet, its canonical parent joined with its name, so a `write` that
* creates a file and a later mutation of it share one key even under a symlinked directory.
*/
async function canonical(env: ExecutionEnv, absolutePath: string, context: Context): Promise<string> {
const result = await env.canonicalPath(absolutePath, context);
if (result.ok) return result.value;
if (result.error.code === "not_supported") return absolutePath;
if (result.error.code !== "not_found") throw result.error;
// The file system splits the path, so a name may contain characters that are separators elsewhere.
const parent = getOrThrow(await env.joinPath([absolutePath, ".."], context));
if (parent === absolutePath || !absolutePath.startsWith(parent)) return absolutePath;
const name = absolutePath.slice(parent.length + (/[/\\]$/.test(parent) ? 0 : 1));
return getOrThrow(await env.joinPath([await canonical(env, parent, context), name], context));
}
/**
* Serialize `edit` and `write` mutations of one file within this process: same file system id and canonical path,
* whichever environment object the call got. Other files, and other file systems, never wait. Concurrent calls on one
* file run in the order their keys resolve. Not a lock against `bash` or other processes.
*/
export async function withFileMutationQueue<T>(
env: ExecutionEnv,
path: string,
fn: () => Promise<T>,
context: Context,
): Promise<T> {
const key = await mutationKey(env, path, context);
// Take the slot without awaiting, so no other call can take it in between.
const previous = queues.get(key) ?? Promise.resolve();
let release = (): void => {};
const done = new Promise<void>((resolve) => {
release = resolve;
});
const tail = previous.then(() => done);
queues.set(key, tail);
await previous;
try {
return await fn();
} finally {
release();
if (queues.get(key) === tail) queues.delete(key);
}
}

58
third_party/pi/packages/durable/src/tools/image-processor.ts vendored Normal file
View File

@@ -0,0 +1,58 @@
/** Limits an image the model sees must keep; the defaults below are the coding agent's. */
export type ImageLimits = {
/** Pixels. Default 2000. */
readonly maxWidth: number;
/** Pixels. Default 2000. */
readonly maxHeight: number;
/** Bytes of base64. Default 4.5 MiB, below Anthropic's 5 MB. */
readonly maxBytes: number;
};
export const DEFAULT_IMAGE_LIMITS: ImageLimits = { maxWidth: 2000, maxHeight: 2000, maxBytes: 4.5 * 1024 * 1024 };
/** An image the model can take: a supported format, within the limits, as base64. */
export type PreparedImage = {
readonly data: string;
readonly mimeType: string;
/** Set when the image was resized: its size before and after, for the model to map coordinates back. */
readonly resized?: {
readonly from: { readonly width: number; readonly height: number };
readonly to: { readonly width: number; readonly height: number };
};
/** The original format, set when it differs from `mimeType`: BMP becomes PNG, a large PNG may become JPEG. */
readonly convertedFrom?: string;
};
/**
* Decodes, orients, resizes, and re-encodes images so the model can take them. `read` uses one when given; without
* one, it passes supported images within the byte limit through as they are. `@earendil-works/pi-durable/images`
* provides a Photon-based one.
*/
export type ImageProcessor = {
/**
* The image, prepared to fit `limits`, or `undefined` when it cannot be decoded or made to fit. A promise, so a
* processor can do the work off the event loop.
*/
prepare(bytes: Uint8Array, mimeType: string, limits: ImageLimits): Promise<PreparedImage | undefined>;
};
/** Formats every provider takes inline; others, such as BMP, need converting. */
export const INLINE_IMAGE_TYPES: ReadonlySet<string> = new Set(["image/png", "image/jpeg", "image/gif", "image/webp"]);
/**
* Standard base64 of `bytes`. `btoa` takes a binary string, built 8 KiB at a time: a single `String.fromCharCode`
* call with megabytes of arguments overflows the stack.
*/
export function toBase64(bytes: Uint8Array): string {
const parts: string[] = [];
for (let i = 0; i < bytes.length; i += 8192) {
// `apply` takes any array-like, though its type asks for an array.
parts.push(String.fromCharCode.apply(null, bytes.subarray(i, i + 8192) as unknown as number[]));
}
return btoa(parts.join(""));
}
/** The length of the base64 of `byteLength` bytes. */
export function base64Length(byteLength: number): number {
return Math.ceil(byteLength / 3) * 4;
}

182
third_party/pi/packages/durable/src/tools/image.ts vendored Normal file
View File

@@ -0,0 +1,182 @@
const PNG_SIGNATURE = [0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0x1a, 0x0a];
/** Bytes every check except the APNG chunk walk needs: BMP reads up to offset 29. */
const HEADER_BYTES = 32;
const BLOCK_BYTES = 64 * 1024;
/** Positional reads of a file of `size` bytes. */
export type ByteSource = {
readonly size: number;
read(offset: number, length: number): Promise<Uint8Array>;
};
/**
* `detectSupportedImageMimeType` of a whole file, reading only its header and, for PNG, the chunk headers up to the
* first `acTL` or `IDAT`.
*/
export async function detectSupportedImageMimeTypeOf(source: ByteSource): Promise<string | undefined> {
const header = await source.read(0, HEADER_BYTES);
if (!startsWith(header, PNG_SIGNATURE)) return detectSupportedImageMimeType(header);
return isPng(header) && !(await isAnimatedPngOf(source)) ? "image/png" : undefined;
}
/** `isAnimatedPng` over a file read in blocks. */
async function isAnimatedPngOf(source: ByteSource): Promise<boolean> {
let block: Uint8Array = new Uint8Array(0);
let blockStart = 0;
const bytesAt = async (offset: number, length: number): Promise<Uint8Array> => {
if (offset < blockStart || offset + length > blockStart + block.length) {
blockStart = offset;
block = await source.read(offset, BLOCK_BYTES);
}
return block.subarray(offset - blockStart, offset - blockStart + length);
};
let offset = PNG_SIGNATURE.length;
while (offset + 8 <= source.size) {
const chunkHeader = await bytesAt(offset, 8);
const chunkLength = readUint32BE(chunkHeader, 0);
if (startsWithAscii(chunkHeader, 4, "acTL")) return true;
if (startsWithAscii(chunkHeader, 4, "IDAT")) return false;
const nextOffset = offset + 8 + chunkLength + 4;
if (nextOffset <= offset || nextOffset > source.size) return false;
offset = nextOffset;
}
return false;
}
export function detectSupportedImageMimeType(buffer: Uint8Array): string | undefined {
if (startsWith(buffer, [0xff, 0xd8, 0xff])) return buffer[3] === 0xf7 ? undefined : "image/jpeg";
if (startsWith(buffer, PNG_SIGNATURE)) return isPng(buffer) && !isAnimatedPng(buffer) ? "image/png" : undefined;
if (startsWithAscii(buffer, 0, "GIF87a") || startsWithAscii(buffer, 0, "GIF89a")) return "image/gif";
if (startsWithAscii(buffer, 0, "RIFF") && startsWithAscii(buffer, 8, "WEBP")) return "image/webp";
if (startsWithAscii(buffer, 0, "BM") && isBmp(buffer)) return "image/bmp";
return undefined;
}
function isPng(buffer: Uint8Array): boolean {
return (
buffer.length >= 16 && readUint32BE(buffer, PNG_SIGNATURE.length) === 13 && startsWithAscii(buffer, 12, "IHDR")
);
}
function isAnimatedPng(buffer: Uint8Array): boolean {
let offset = PNG_SIGNATURE.length;
while (offset + 8 <= buffer.length) {
const chunkLength = readUint32BE(buffer, offset);
const chunkTypeOffset = offset + 4;
if (startsWithAscii(buffer, chunkTypeOffset, "acTL")) return true;
if (startsWithAscii(buffer, chunkTypeOffset, "IDAT")) return false;
const nextOffset = offset + 8 + chunkLength + 4;
if (nextOffset <= offset || nextOffset > buffer.length) return false;
offset = nextOffset;
}
return false;
}
/**
* The width and height an image of `mimeType` declares in its header, without decoding it: PNG's IHDR, GIF's screen,
* WebP's VP8, VP8L, or VP8X chunk, JPEG's first SOF segment. `undefined` when the header cannot be read.
*/
export function imageDimensions(
bytes: Uint8Array,
mimeType: string,
): { readonly width: number; readonly height: number } | undefined {
switch (mimeType) {
case "image/png":
return bytes.length < 24 ? undefined : { width: readUint32BE(bytes, 16), height: readUint32BE(bytes, 20) };
case "image/gif":
return bytes.length < 10 ? undefined : { width: readUint16LE(bytes, 6), height: readUint16LE(bytes, 8) };
case "image/webp":
if (bytes.length < 30) return undefined;
if (startsWithAscii(bytes, 12, "VP8 "))
return { width: readUint16LE(bytes, 26) & 0x3fff, height: readUint16LE(bytes, 28) & 0x3fff };
if (startsWithAscii(bytes, 12, "VP8L")) {
const bits = readUint32LE(bytes, 21);
return { width: (bits & 0x3fff) + 1, height: ((bits >>> 14) & 0x3fff) + 1 };
}
if (startsWithAscii(bytes, 12, "VP8X")) {
const uint24 = (offset: number): number => readUint16LE(bytes, offset) + ((bytes[offset + 2] ?? 0) << 16);
return { width: uint24(24) + 1, height: uint24(27) + 1 };
}
return undefined;
case "image/jpeg":
for (let offset = 2; offset + 9 <= bytes.length; ) {
if (bytes[offset] !== 0xff) return undefined;
const marker = bytes[offset + 1]!;
if (marker === 0xff) {
offset++;
continue;
}
// SOF0 to SOF15, except DHT (C4), JPG (C8), and DAC (CC), which share the range.
if (marker >= 0xc0 && marker <= 0xcf && marker !== 0xc4 && marker !== 0xc8 && marker !== 0xcc) {
return { width: readUint16BE(bytes, offset + 7), height: readUint16BE(bytes, offset + 5) };
}
offset += 2 + readUint16BE(bytes, offset + 2);
}
return undefined;
default:
return undefined;
}
}
function isBmp(buffer: Uint8Array): boolean {
if (buffer.length < 26) return false;
const declaredFileSize = readUint32LE(buffer, 2);
const pixelDataOffset = readUint32LE(buffer, 10);
const dibHeaderSize = readUint32LE(buffer, 14);
if (declaredFileSize !== 0 && declaredFileSize < 26) return false;
if (pixelDataOffset < 14 + dibHeaderSize) return false;
if (declaredFileSize !== 0 && pixelDataOffset >= declaredFileSize) return false;
let colorPlanes: number;
let bitsPerPixel: number;
if (dibHeaderSize === 12) {
colorPlanes = readUint16LE(buffer, 22);
bitsPerPixel = readUint16LE(buffer, 24);
} else if (dibHeaderSize >= 40 && dibHeaderSize <= 124) {
if (buffer.length < 30) return false;
colorPlanes = readUint16LE(buffer, 26);
bitsPerPixel = readUint16LE(buffer, 28);
} else {
return false;
}
return colorPlanes === 1 && [1, 4, 8, 16, 24, 32].includes(bitsPerPixel);
}
function readUint16LE(buffer: Uint8Array, offset: number): number {
return (buffer[offset] ?? 0) + ((buffer[offset + 1] ?? 0) << 8);
}
function readUint16BE(buffer: Uint8Array, offset: number): number {
return ((buffer[offset] ?? 0) << 8) + (buffer[offset + 1] ?? 0);
}
function readUint32BE(buffer: Uint8Array, offset: number): number {
return (
(buffer[offset] ?? 0) * 0x1000000 +
((buffer[offset + 1] ?? 0) << 16) +
((buffer[offset + 2] ?? 0) << 8) +
(buffer[offset + 3] ?? 0)
);
}
function readUint32LE(buffer: Uint8Array, offset: number): number {
return (
(buffer[offset] ?? 0) +
((buffer[offset + 1] ?? 0) << 8) +
((buffer[offset + 2] ?? 0) << 16) +
(buffer[offset + 3] ?? 0) * 0x1000000
);
}
function startsWith(buffer: Uint8Array, bytes: number[]): boolean {
if (buffer.length < bytes.length) return false;
return bytes.every((byte, index) => buffer[index] === byte);
}
function startsWithAscii(buffer: Uint8Array, offset: number, text: string): boolean {
if (buffer.length < offset + text.length) return false;
for (let index = 0; index < text.length; index++) {
if (buffer[offset + index] !== text.charCodeAt(index)) return false;
}
return true;
}

39
third_party/pi/packages/durable/src/tools/index.ts vendored Normal file
View File

@@ -0,0 +1,39 @@
import { defineExtension } from "../harness/define.ts";
import { createBashTool } from "./bash.ts";
import { createEditTool } from "./edit.ts";
import { createReadTool, type ReadToolOptions } from "./read.ts";
import { createWriteTool } from "./write.ts";
export {
type BashExecution,
type BashPrepare,
type BashToolInput,
type BashToolOptions,
createBashTool,
createPowerShellTool,
type PowerShellToolInput,
type PowerShellToolOptions,
} from "./bash.ts";
export { createEditTool, type EditToolDetails, type EditToolInput } from "./edit.ts";
export {
DEFAULT_IMAGE_LIMITS,
type ImageLimits,
type ImageProcessor,
type PreparedImage,
} from "./image-processor.ts";
export { createReadTool, type ReadToolDetails, type ReadToolInput, type ReadToolOptions } from "./read.ts";
export { createWriteTool, type WriteToolInput } from "./write.ts";
/**
* The `coding-tools` extension: `read`, `write`, `edit`, and `bash`; nothing installs it automatically.
* `createPowerShellTool()` adds `powershell`. `images` prepares the images `read` returns (see `ReadToolOptions`).
*/
export function createCodingTools(options: ReadToolOptions = {}) {
return defineExtension({
name: "coding-tools",
tools: [createReadTool(options), createWriteTool(), createEditTool(), createBashTool()],
});
}
/** `createCodingTools()` without an image processor. */
export const CodingTools = createCodingTools();

30
third_party/pi/packages/durable/src/tools/path-utils.ts vendored Normal file
View File

@@ -0,0 +1,30 @@
import type { Context } from "@earendil-works/chord";
import { type ExecutionEnv, getOrThrow } from "../env/index.ts";
const UNICODE_SPACES = /[\u00A0\u2000-\u200A\u202F\u205F\u3000]/g;
const NARROW_NO_BREAK_SPACE = "\u202F";
function normalizeToolPath(path: string): string {
const normalized = path.replace(UNICODE_SPACES, " ");
return normalized.startsWith("@") ? normalized.slice(1) : normalized;
}
export async function resolveToolPath(env: ExecutionEnv, path: string, context: Context): Promise<string> {
return getOrThrow(await env.absolutePath(normalizeToolPath(path), context));
}
export async function resolveReadToolPath(env: ExecutionEnv, path: string, context: Context): Promise<string> {
const resolved = await resolveToolPath(env, path, context);
const variants = [
resolved,
resolved.replace(/ (AM|PM)\./gi, `${NARROW_NO_BREAK_SPACE}$1.`),
resolved.normalize("NFD"),
resolved.replace(/'/g, "\u2019"),
resolved.normalize("NFD").replace(/'/g, "\u2019"),
];
for (const variant of new Set(variants)) {
if (getOrThrow(await env.exists(variant, context))) return variant;
}
return resolved;
}

312
third_party/pi/packages/durable/src/tools/read.ts vendored Normal file
View File

@@ -0,0 +1,312 @@
import type { Context } from "@earendil-works/chord";
import { characterEnd } from "@earendil-works/chord/output";
import { type Static, Type } from "typebox";
import {
type BinaryReader,
type FileInfo,
getOrThrow,
type LineScan,
rangeDecoder,
startsWithBom,
} from "../env/index.ts";
import { defineTool } from "../harness/define.ts";
import type { ToolDiagnostic, ToolExecutionApi, ToolExecutionResult, ToolRegistration } from "../harness/types.ts";
import {
DEFAULT_MAX_BYTES,
DEFAULT_MAX_LINES,
formatSize,
type TruncationResult,
truncateHeadOf,
utf8ByteLength,
} from "../truncate.ts";
import { requireEnv } from "./env.ts";
import { detectSupportedImageMimeTypeOf, imageDimensions } from "./image.ts";
import {
base64Length,
DEFAULT_IMAGE_LIMITS,
type ImageLimits,
type ImageProcessor,
INLINE_IMAGE_TYPES,
toBase64,
} from "./image-processor.ts";
import { resolveReadToolPath } from "./path-utils.ts";
const readSchema = Type.Object({
path: Type.String({ description: "Path to the file to read (relative or absolute)" }),
offset: Type.Optional(Type.Number({ description: "Line number to start reading from (1-indexed)" })),
limit: Type.Optional(Type.Number({ description: "Maximum number of lines to read" })),
});
export type ReadToolInput = Static<typeof readSchema>;
export type ReadToolDetails = {
/** How the shown text was cut; the text itself is the result content. */
truncation?: Omit<TruncationResult, "content">;
};
const READ_CHUNK = 64 * 1024;
/** `Array.prototype.slice`'s conversion of an index: NaN is 0, other values truncate toward zero. */
function sliceIndex(value: number): number {
return Number.isNaN(value) ? 0 : Math.trunc(value);
}
/**
* The decoded start of bytes `[start, end)` of the file, decoded as part of the whole file: all of it, or enough for
* `truncateHeadOf` (more than `DEFAULT_MAX_BYTES + 1` bytes, or `DEFAULT_MAX_LINES` newlines).
*/
async function readHead(
reader: BinaryReader,
start: number,
end: number,
skipBom: boolean,
context: Context,
): Promise<string> {
const decoder = rangeDecoder();
let text = "";
let newlines = 0;
for (let position = skipBom && start === 0 ? 3 : start; position < end; ) {
const bytes = getOrThrow(await reader.read(position, Math.min(READ_CHUNK, end - position), context));
if (bytes.length === 0) break;
position += bytes.length;
const decoded = decoder.decode(bytes, { stream: true });
text += decoded;
for (let index = decoded.indexOf("\n"); index !== -1; index = decoded.indexOf("\n", index + 1)) newlines++;
if (newlines >= DEFAULT_MAX_LINES || utf8ByteLength(text) > DEFAULT_MAX_BYTES + 1) return text;
}
return text + decoder.decode();
}
export type ReadToolOptions = {
/** Resizes and converts images; without one, images within `DEFAULT_IMAGE_LIMITS.maxBytes` pass through as they are. */
readonly images?: ImageProcessor;
};
/**
* Reads text files and images. Remarks about truncation, continuation, and image processing are diagnostics; the content
* is only file text, or one image, which programs get as its `ImageContent`.
*/
export function createReadTool(options: ReadToolOptions = {}): ToolRegistration<typeof readSchema, ReadToolDetails> {
const formats = options.images === undefined ? "jpg, png, gif, webp" : "jpg, png, gif, webp, bmp";
return defineTool({
name: "read",
description: `Read the contents of a file. Supports text files and images (${formats}). Images are sent as attachments. For text files, output is truncated to ${DEFAULT_MAX_LINES} lines or ${DEFAULT_MAX_BYTES / 1024}KB (whichever is hit first). Use offset/limit for large files. When you need the full file, continue with offset until complete.`,
parameters: readSchema,
async execute(args, api, context) {
const { path, offset, limit } = args;
const env = requireEnv(api);
const absolutePath = await resolveReadToolPath(env, path, context);
const reader = getOrThrow(await env.openBinaryReader(absolutePath, undefined, context));
try {
// A concurrent writer can change the file between the scan and the reads. Appending (a growing log) leaves
// the scanned text as it was; an image must be whole. Otherwise the file is read again once.
for (let attempt = 0; ; attempt++) {
const before = getOrThrow(await reader.info(context));
const mimeType = await detectSupportedImageMimeTypeOf({
size: before.size,
read: async (position, length) => getOrThrow(await reader.read(position, length, context)),
});
const result =
mimeType === undefined
? await readText(reader, path, offset, limit, context)
: await readImage(
reader,
before,
path,
mimeType,
options.images,
await imageModel(api, context),
context,
);
const after = getOrThrow(await reader.info(context));
const unchanged = after.size === before.size && after.mtimeMs === before.mtimeMs;
if (unchanged || (mimeType === undefined && after.size > before.size)) return result;
if (attempt === 1) throw new Error(`${path} changed while it was read`);
}
} finally {
await reader.close(context);
}
},
});
}
/**
* One image block, prepared by `processor` or, without one, as it is when its format is inline and it fits the limits
* (its size is checked before reading it, its dimensions from its header); otherwise an error result saying why. What happened to the image is an `info` diagnostic, so programs get the
* bare `ImageContent`. A model without image input sees pi-ai's placeholder instead; a diagnostic says so.
*/
async function readImage(
reader: BinaryReader,
info: FileInfo,
path: string,
mimeType: string,
processor: ImageProcessor | undefined,
{ vision, limits }: ImageModel,
context: Context,
): Promise<ToolExecutionResult<ReadToolDetails>> {
let image: { readonly data: string; readonly mimeType: string };
const notes: string[] = [];
if (processor !== undefined) {
const bytes = getOrThrow(await reader.read(0, info.size, context));
const prepared = await processor.prepare(bytes, mimeType, limits);
if (prepared === undefined)
return imageError(`${path} is an image (${mimeType}) that cannot be prepared for the model`);
image = prepared;
if (prepared.convertedFrom !== undefined)
notes.push(`Converted from ${prepared.convertedFrom} to ${prepared.mimeType}.`);
if (prepared.resized !== undefined) {
const { from, to } = prepared.resized;
const scale = (from.width / to.width).toFixed(2);
notes.push(
`Resized from ${from.width}x${from.height} to ${to.width}x${to.height}. Multiply coordinates by ${scale} to map them to the original.`,
);
}
} else {
if (!INLINE_IMAGE_TYPES.has(mimeType)) {
return imageError(
`${path} is an image (${mimeType}) that needs converting, and no image processor is configured`,
);
}
const unshrinkable = "and no image processor is configured to shrink it";
if (base64Length(info.size) > limits.maxBytes) {
return imageError(
`${path} is an image (${mimeType}) of ${formatSize(info.size)}, too large to send, ${unshrinkable}`,
);
}
const bytes = getOrThrow(await reader.read(0, info.size, context));
const size = imageDimensions(bytes, mimeType);
if (size === undefined) return imageError(`${path} is an image (${mimeType}) whose size cannot be read`);
if (size.width > limits.maxWidth || size.height > limits.maxHeight) {
return imageError(
`${path} is an image (${mimeType}) of ${size.width}x${size.height}, larger than ${limits.maxWidth}x${limits.maxHeight}, ${unshrinkable}`,
);
}
image = { data: toBase64(bytes), mimeType };
}
if (!vision) notes.push("The current model does not support images; it sees a placeholder instead.");
const diagnostics: ToolDiagnostic[] = [
{ severity: "info", code: "image", message: [`Read image file [${image.mimeType}].`, ...notes].join(" ") },
];
return { output: [{ type: "image", data: image.data, mimeType: image.mimeType }], diagnostics };
}
/** What the conversation's model takes: whether images, and within which limits. */
type ImageModel = { readonly vision: boolean; readonly limits: ImageLimits };
/**
* The conversation model's image input: `DEFAULT_IMAGE_LIMITS` overridden by its `inputLimits.images.resize`. Without
* a resolvable model, it counts as one that takes images within the defaults.
*/
async function imageModel(api: ToolExecutionApi, context: Context): Promise<ImageModel> {
const ref = (await api.agent(context)).model;
const model = ref === undefined ? undefined : api.models.getModel(ref.provider, ref.modelId);
const resize = model?.inputLimits?.images?.resize;
return {
vision: model?.input.includes("image") ?? true,
limits: {
maxWidth: resize?.maxWidth ?? DEFAULT_IMAGE_LIMITS.maxWidth,
maxHeight: resize?.maxHeight ?? DEFAULT_IMAGE_LIMITS.maxHeight,
maxBytes: resize?.maxBytes ?? DEFAULT_IMAGE_LIMITS.maxBytes,
},
};
}
function imageError(message: string): ToolExecutionResult<ReadToolDetails> {
return { output: [], isError: true, diagnostics: [{ severity: "error", code: "unsupported_image", message }] };
}
/**
* The read result for the opened file. It equals decoding the whole file with `TextDecoder`, splitting it on `\n`,
* and bounding the selected lines with `truncateHead`, while reading only one scan's worth of the file plus the head.
*/
async function readText(
reader: BinaryReader,
path: string,
offset: number | undefined,
limit: number | undefined,
context: Context,
) {
const startLine = offset ? Math.max(0, offset - 1) : 0;
const startLineDisplay = startLine + 1;
// Lines are selected like `allLines.slice(startLine, endLine)`, which truncates fractional indices.
const sliceStart = sliceIndex(startLine);
// One pass finds the line count and the selection; a selection past the last line ends with it, as `slice` does,
// and an empty one (a zero or negative limit) is scanned as one line and then ignored.
// A start beyond any file is scanned from 0 only to count lines; the offset check below then fails as before.
const scanStart = Number.isSafeInteger(sliceStart) ? sliceStart : 0;
const requestedEnd = limit === undefined ? undefined : Math.max(scanStart + 1, sliceIndex(startLine + limit));
const scanEnd = requestedEnd !== undefined && Number.isSafeInteger(requestedEnd) ? requestedEnd : undefined;
const scanOf = async (endLine: number | undefined): Promise<LineScan> =>
getOrThrow(
await reader.scanLines({ startLine: scanStart, ...(endLine === undefined ? {} : { endLine }) }, context),
);
let scan = await scanOf(scanEnd);
const totalFileLines = scan.newlines + 1;
if (startLine >= totalFileLines) {
throw new Error(`Offset ${offset} is beyond end of file (${totalFileLines} lines total)`);
}
let userLimitedLines: number | undefined;
let selectedLineCount = totalFileLines - sliceStart;
if (limit !== undefined) {
const endLine = Math.min(startLine + limit, totalFileLines);
userLimitedLines = endLine - startLine;
// `slice` counts a negative end from the end of the lines, which only the line count tells; scan again for it.
const relativeEnd = sliceIndex(endLine);
const sliceEnd = relativeEnd < 0 ? Math.max(totalFileLines + relativeEnd, 0) : relativeEnd;
selectedLineCount = Math.max(0, sliceEnd - sliceStart);
if (selectedLineCount > 0 && relativeEnd < 0) scan = await scanOf(sliceEnd);
}
const empty = selectedLineCount === 0;
// Counted like `truncateHead`: a trailing newline adds no line, and empty text has none.
const endsWithNewline = !empty && scan.lastLineStart === scan.end && scan.lastLineStart > scan.start;
const totals = {
lines: empty || scan.selectedBytes === 0 ? 0 : selectedLineCount - (endsWithNewline ? 1 : 0),
bytes: empty ? 0 : scan.selectedBytes,
};
const firstBytes = getOrThrow(await reader.read(0, 3, context));
const head = empty ? "" : await readHead(reader, scan.start, scan.end, startsWithBom(firstBytes), context);
const { content: headText, ...truncation } = truncateHeadOf(head, totals);
const diagnostics: ToolDiagnostic[] = [];
let outputText = headText;
let details: ReadToolDetails | undefined;
if (truncation.firstLineExceedsLimit) {
// Show the start of the line, cut at the byte limit on a character boundary. Like `allLines[startLine]`, a
// fractional start line names no line.
const integral = Number.isInteger(startLine);
const lineBytes = new TextEncoder().encode(integral ? (head.split("\n")[0] ?? "") : "");
const lineSize = integral ? scan.firstLineBytes : 0;
const end = characterEnd(lineBytes, DEFAULT_MAX_BYTES);
outputText = new TextDecoder().decode(lineBytes.subarray(0, end));
diagnostics.push({
severity: "warn",
code: "truncated",
message: `Line ${startLineDisplay} is ${formatSize(lineSize)}, exceeds the ${formatSize(DEFAULT_MAX_BYTES)} limit; showing its first ${formatSize(end)}. Use bash: sed -n '${startLineDisplay}p' ${path} | tail -c +${end + 1}`,
});
details = { truncation: { ...truncation, outputBytes: end, outputLines: 1 } };
} else if (truncation.truncated) {
const endLineDisplay = startLineDisplay + truncation.outputLines - 1;
const nextOffset = endLineDisplay + 1;
const limitText = truncation.truncatedBy === "lines" ? "" : ` (${formatSize(DEFAULT_MAX_BYTES)} limit)`;
diagnostics.push({
severity: "info",
code: "truncated",
message: `Showing lines ${startLineDisplay}-${endLineDisplay} of ${totalFileLines}${limitText}. Use offset=${nextOffset} to continue.`,
});
details = { truncation };
} else if (userLimitedLines !== undefined && startLine + userLimitedLines < totalFileLines) {
const remaining = totalFileLines - (startLine + userLimitedLines);
const nextOffset = startLine + userLimitedLines + 1;
diagnostics.push({
severity: "info",
message: `${remaining} more lines in file. Use offset=${nextOffset} to continue.`,
});
}
return {
output: outputText === "" ? [] : [{ type: "text" as const, text: outputText }],
...(details === undefined ? {} : { details }),
diagnostics,
};
}

39
third_party/pi/packages/durable/src/tools/write.ts vendored Normal file
View File

@@ -0,0 +1,39 @@
import { type Static, Type } from "typebox";
import { getOrThrow } from "../env/index.ts";
import { defineTool } from "../harness/define.ts";
import type { ToolRegistration } from "../harness/types.ts";
import { requireEnv } from "./env.ts";
import { withFileMutationQueue } from "./file-mutation-queue.ts";
import { resolveToolPath } from "./path-utils.ts";
const writeSchema = Type.Object({
path: Type.String({ description: "Path to the file to write (relative or absolute)" }),
content: Type.String({ description: "Content to write to the file" }),
});
export type WriteToolInput = Static<typeof writeSchema>;
export function createWriteTool(): ToolRegistration<typeof writeSchema> {
return defineTool({
name: "write",
description:
"Write content to a file. Creates the file if it doesn't exist, overwrites if it does. Automatically creates parent directories.",
parameters: writeSchema,
async execute(args, api, context) {
const { path, content } = args;
const env = requireEnv(api);
const absolutePath = await resolveToolPath(env, path, context);
return withFileMutationQueue(
env,
absolutePath,
async () => {
if (context.abortSignal?.aborted) throw new Error("Operation aborted");
getOrThrow(await env.writeFile(absolutePath, content, context));
if (context.abortSignal?.aborted) throw new Error("Operation aborted");
return { output: [{ type: "text", text: `Successfully wrote to ${path}` }] };
},
context,
);
},
});
}