chore(vendor): import pi 1.3.0

Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
2026-10-15 17:58:48 -04:00
parent 719370c8f7
commit 6081e3e325
1743 changed files with 205460 additions and 131705 deletions

View File

@@ -0,0 +1,45 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { registerEnvConformance } from "@earendil-works/pi-durable/testing";
import { afterAll, describe, expect, it } from "vitest";
import { Connection } from "../src/connection.ts";
import { LocalExecutionEnv } from "../src/list-files.ts";
import { RemoteExecutionEnv } from "../src/remote-env.ts";
import { daemon } from "./daemon.ts";
const connection = new Connection({ command: [daemon] });
afterAll(() => connection.close());
const windows = process.platform === "win32";
const gitBash = join(process.env.ProgramFiles ?? "C:\\Program Files", "Git", "bin", "bash.exe");
registerEnvConformance(
{ describe, expect, it },
"RemoteExecutionEnv over a pipe",
async (use) => {
const cwd = mkdtempSync(join(tmpdir(), "pi-env-conformance-"));
try {
await use(new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd, watch: { pollIntervalMs: 100 } }));
} finally {
// On Windows a killed command's processes can hold the directory for a moment.
rmSync(cwd, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 });
}
},
// Git Bash's `ln -s` copies instead of linking unless native symlinks are enabled.
windows ? { shell: [gitBash, "-c"], symlinks: false, listFiles: true } : { listFiles: true },
);
registerEnvConformance(
{ describe, expect, it },
"LocalExecutionEnv",
async (use) => {
const cwd = mkdtempSync(join(tmpdir(), "pi-env-local-conformance-"));
try {
await use(new LocalExecutionEnv({ connection, cwd, watch: { pollIntervalMs: 100 } }));
} finally {
rmSync(cwd, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 });
}
},
windows ? { shell: [gitBash, "-c"], symlinks: false, listFiles: true } : { listFiles: true },
);

View File

@@ -0,0 +1,13 @@
import { existsSync } from "node:fs";
import { resolve } from "node:path";
/**
* The daemon the tests run: `PI_ENV_DAEMON` (CI points it at release builds), else the one `cargo build` built in
* ../daemon.
*/
export const daemon =
process.env.PI_ENV_DAEMON ??
resolve(import.meta.dirname, `../daemon/target/debug/pi-env${process.platform === "win32" ? ".exe" : ""}`);
if (!existsSync(daemon)) {
throw new Error(`Build the daemon first: npm run build:daemon in packages/env (missing ${daemon})`);
}

View File

@@ -0,0 +1,361 @@
import { spawn } from "node:child_process";
import { mkdtempSync, realpathSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import type { Context } from "@earendil-works/chord";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import type { ToolExecutionApi } from "@earendil-works/pi-durable";
import type { ExecutionEnv, FileError, Result } from "@earendil-works/pi-durable/env";
import { NodeExecutionEnv } from "@earendil-works/pi-durable/env/node";
import { createBashTool, createReadTool } from "@earendil-works/pi-durable/tools";
import { afterAll, describe, expect, it } from "vitest";
import { Connection } from "../src/connection.ts";
import { RemoteExecutionEnv } from "../src/remote-env.ts";
import { daemon } from "./daemon.ts";
const connection = new Connection({ command: [daemon] });
const context: Context = BACKGROUND_CONTEXT;
const dirs: string[] = [];
afterAll(() => {
connection.close();
for (const dir of dirs) rmSync(dir, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 });
});
function pair(): { local: ExecutionEnv; remote: ExecutionEnv; roots: [string, string] } {
const localRoot = mkdtempSync(join(tmpdir(), "pi-env-local-"));
const remoteRoot = mkdtempSync(join(tmpdir(), "pi-env-remote-"));
dirs.push(localRoot, remoteRoot);
return {
local: new NodeExecutionEnv({ cwd: localRoot }),
remote: new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd: remoteRoot }),
roots: [localRoot, remoteRoot],
};
}
function random(seed: number): () => number {
let state = seed >>> 0;
return () => {
state = (state + 0x6d2b79f5) >>> 0;
let t = state;
t = Math.imul(t ^ (t >>> 15), t | 1);
t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
};
}
/** A result with the environment's root replaced and only the error code of failures, for comparison. */
function normalize(value: unknown, root: string): unknown {
const json = JSON.stringify(value, (key, field: unknown) => {
if (field instanceof Uint8Array) return { bytes: Buffer.from(field).toString("hex") };
if (key === "mtimeMs") return "mtime";
if (field && typeof field === "object" && "ok" in field && (field as Result<unknown, FileError>).ok === false) {
return { ok: false, code: (field as unknown as { error: FileError }).error.code };
}
return field;
});
if (json === undefined) return undefined;
// Roots appear JSON-escaped (backslashes doubled on Windows), and resolved paths spell them without Windows' 8.3
// short names (RUNNER~1).
const escaped = (path: string) => JSON.stringify(path).slice(1, -1);
return JSON.parse(json.replaceAll(escaped(realpathSync.native(root)), "<root>").replaceAll(escaped(root), "<root>"));
}
const NAMES = ["a.txt", "b", "dir", "dir/c.txt", "dir/sub", "dir/sub/d.bin", "missing", "a.txt/x"];
const CONTENTS: (string | Uint8Array)[] = [
"",
"hello\n",
"\ufeffbom\r\nline",
Uint8Array.from([0xe2, 0x82, 0x0a, 0xff, 0xef, 0xbb, 0xbf]),
"x".repeat(70_000),
// Crosses write chunks, read chunks and multi-byte characters split between them.
"é".repeat(300_001),
];
type Operation = (env: ExecutionEnv) => Promise<unknown>;
/** A random operation with its arguments fixed, and a label for failure messages. */
function randomOperation(next: () => number): { label: string; run: Operation } {
const args: unknown[] = [];
const pick = <T>(value: T): T => {
args.push(
value instanceof Uint8Array
? "<bytes>"
: typeof value === "string" && value.length > 20
? `<${value.length} chars>`
: value,
);
return value;
};
const nameValue = () => pick(NAMES[Math.floor(next() * NAMES.length)]!);
const contentValue = () => pick(CONTENTS[Math.floor(next() * CONTENTS.length)]!);
const flagValue = () => pick(next() < 0.5);
const small = (limit: number) => pick(Math.floor(next() * limit));
const choices: [string, () => Operation][] = [
[
"writeFile",
() => {
const n = nameValue();
const c = contentValue();
return (env) => env.writeFile(n, c, context);
},
],
[
"appendFile",
() => {
const n = nameValue();
const c = contentValue();
return (env) => env.appendFile(n, c, context);
},
],
[
"readTextFile",
() => {
const n = nameValue();
return (env) => env.readTextFile(n, context);
},
],
[
"readBinaryFile",
() => {
const n = nameValue();
return (env) => env.readBinaryFile(n, context);
},
],
[
"readTextLines",
() => {
const n = nameValue();
const m = small(4);
return (env) => env.readTextLines(n, { maxLines: m }, context);
},
],
[
"fileInfo",
() => {
const n = nameValue();
return (env) => env.fileInfo(n, context);
},
],
[
"exists",
() => {
const n = nameValue();
return (env) => env.exists(n, context);
},
],
[
"listDir",
() => {
const n = nameValue();
return (env) => env.listDir(n, context);
},
],
[
"createDir",
() => {
const n = nameValue();
const r = flagValue();
return (env) => env.createDir(n, { recursive: r }, context);
},
],
[
"remove",
() => {
const n = nameValue();
const r = flagValue();
const f = flagValue();
return (env) => env.remove(n, { recursive: r, force: f }, context);
},
],
[
"renameFile",
() => {
const a = nameValue();
const b = nameValue();
return (env) => env.renameFile(a, b, context);
},
],
[
"truncateFile",
() => {
const n = nameValue();
const size = small(10);
return (env) => env.truncateFile(n, size, context);
},
],
[
"flushFile",
() => {
const n = nameValue();
return (env) => env.flushFile(n, context);
},
],
[
"canonicalPath",
() => {
const n = nameValue();
return (env) => env.canonicalPath(n, context);
},
],
[
"openBinaryReader",
() => {
const n = nameValue();
const noFollow = flagValue();
const offset = small(8);
const length = small(8);
return async (env) => {
const reader = await env.openBinaryReader(n, { noFollow }, context);
if (!reader.ok) return reader;
try {
return [
await reader.value.read(offset, length, context),
await reader.value.scanLines({ startLine: 0 }, context),
];
} finally {
await reader.value.close(context);
}
};
},
],
];
const [label, make] = choices[Math.floor(next() * choices.length)]!;
const run = make();
return { label: `${label}(${args.map((arg) => JSON.stringify(arg)).join(", ")})`, run };
}
describe("RemoteExecutionEnv against NodeExecutionEnv", () => {
it("gives the same results for random file operation sequences", async () => {
for (let seed = 1; seed <= 60; seed++) {
const next = random(seed);
const { local, remote, roots } = pair();
for (let step = 0; step < 25; step++) {
const operation = randomOperation(next);
const expected = normalize(await operation.run(local), roots[0]);
const actual = normalize(await operation.run(remote), roots[1]);
expect(actual, `seed ${seed} step ${step} ${operation.label}`).toEqual(expected);
}
}
}, 120_000);
it.skipIf(process.platform === "win32")(
"reads devices, FIFOs and directories like Node",
async () => {
const { local, remote, roots } = pair();
const results = async (env: ExecutionEnv, root: string) => {
const fifo = join(root, "fifo");
const made = await env.exec(["mkfifo", fifo], undefined, context);
if (!made.ok || made.value.exitCode !== 0) throw new Error("mkfifo failed");
await env.createDir("dir", undefined, context);
// A writer for each blocking open of the FIFO.
const write = (text: string) => spawn("sh", ["-c", `printf '%s' "$1" > "$2"`, "sh", text, fifo]);
write("fifo text\n");
const fifoText = await env.readTextFile("fifo", context);
write("fifo bytes");
const fifoBytes = await env.readBinaryFile("fifo", context);
write("x");
const fifoLines = await env.readTextLines("fifo", undefined, context);
const dirReader = await env.openTextLineReader("dir", context);
const dirLine = dirReader.ok ? await dirReader.value.readLine(context) : undefined;
if (dirReader.ok) await dirReader.value.close(context);
return normalize(
{
nullText: await env.readTextFile("/dev/null", context),
nullBytes: await env.readBinaryFile("/dev/null", context),
nullLines: await env.readTextLines("/dev/null", undefined, context),
fifoText,
fifoBytes,
fifoLines,
dirReaderOpened: dirReader.ok,
dirLine,
dirText: await env.readTextFile("dir", context),
dirBytes: await env.readBinaryFile("dir", context),
},
root,
);
};
expect(await results(remote, roots[1])).toEqual(await results(local, roots[0]));
},
30_000,
);
// macOS file systems refuse names that are not UTF-8.
it.skipIf(process.platform !== "linux" && process.platform !== "android")(
"lists names that are not UTF-8 like Node",
async () => {
const { local, remote, roots } = pair();
for (const root of roots) {
for (const name of [[0x62], [0x61, 0xff], [0xc3, 0xa9], [0x61, 0xfe, 0x7a], [0x7a]]) {
writeFileSync(Buffer.concat([Buffer.from(`${root}/`), Buffer.from(name)]), "x");
}
}
const results = async (env: ExecutionEnv, root: string) => {
const reader = await env.openDirReader(".", context);
const pages: unknown[] = [];
if (reader.ok) {
for (let done = false; !done; ) {
const page = await reader.value.next(2, context);
pages.push(page);
done = !page.ok || page.value.done;
}
await reader.value.close(context);
}
return normalize({ list: await env.listDir(".", context), pages }, root);
};
expect(await results(remote, roots[1])).toEqual(await results(local, roots[0]));
},
);
it("gives the same read and bash tool results", async () => {
const { local, remote, roots } = pair();
const files: [string, string | Uint8Array][] = [
["text.txt", "one\ntwo\n\nthree"],
["big.txt", "line\n".repeat(3000)],
["long.txt", `${"é".repeat(40_000)}\nend`],
["bom.txt", "\ufeffhello"],
["bad.txt", Uint8Array.from([0x61, 0xe2, 0x82, 0x0a, 0x62])],
];
for (const [path, content] of files) {
await local.writeFile(path, content, context);
await remote.writeFile(path, content, context);
}
const api = (env: ExecutionEnv, output: string[]) =>
({
env,
outputWindow: undefined,
output: (chunk: string | Uint8Array) => output.push(String(chunk)),
diagnostic: () => {},
details: async () => {},
}) as unknown as ToolExecutionApi;
const run = async (
env: ExecutionEnv,
tool: ReturnType<typeof createReadTool> | ReturnType<typeof createBashTool>,
args: object,
) => {
const output: string[] = [];
try {
const result = await tool.execute(args as never, api(env, output), context);
return { result, output: output.join("") };
} catch (error) {
return { error: (error as Error).message, output: output.join("") };
}
};
const cases: [ReturnType<typeof createReadTool> | ReturnType<typeof createBashTool>, object][] = [
[createReadTool(), { path: "text.txt" }],
[createReadTool(), { path: "text.txt", offset: 2, limit: 1 }],
[createReadTool(), { path: "big.txt" }],
[createReadTool(), { path: "long.txt" }],
[createReadTool(), { path: "bom.txt" }],
[createReadTool(), { path: "bad.txt" }],
[createReadTool(), { path: "missing.txt" }],
[createBashTool(), { command: "cat text.txt; echo err >&2; exit 0" }],
[createBashTool(), { command: "exit 3" }],
[createBashTool(), { command: "printf 'a\\nb'; ls" }],
];
for (const [tool, args] of cases) {
const expected = normalize(await run(local, tool, args), roots[0]);
const actual = normalize(await run(remote, tool, args), roots[1]);
expect(actual, JSON.stringify(args)).toEqual(expected);
}
}, 60_000);
});

View File

@@ -0,0 +1,48 @@
/**
* A stand-in for a daemon released before `hello.features`: it answers `hello` without `features` and refuses every
* other operation as an unknown one, as such a daemon refuses `listFiles`. Started as `node old-daemon.ts serve --token
* <hex>`.
*/
const token = process.argv[process.argv.indexOf("--token") + 1];
process.stdout.write(`PI-ENV ${token}\n`);
function send(type: number, id: number, json: Record<string, unknown>): void {
const body = Buffer.from(JSON.stringify(json), "utf8");
const header = Buffer.alloc(13);
header.writeUInt32BE(9 + body.length, 0);
header.writeUInt8(type, 4);
header.writeUInt32BE(id, 5);
header.writeUInt32BE(body.length, 9);
process.stdout.write(Buffer.concat([header, body]));
}
let buffered = Buffer.alloc(0);
process.stdin.on("data", (chunk: Buffer) => {
buffered = Buffer.concat([buffered, chunk]);
while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE(0)) {
const length = buffered.readUInt32BE(0);
const type = buffered.readUInt8(4);
const id = buffered.readUInt32BE(5);
const jsonLength = buffered.readUInt32BE(9);
const json = JSON.parse(buffered.subarray(13, 13 + jsonLength).toString("utf8")) as { op?: string };
buffered = buffered.subarray(4 + length);
if (type !== 1) continue;
if (json.op === "hello") {
send(2, id, {
protocol: 1,
version: "1.1.0",
os: "linux",
arch: "x86_64",
home: "/",
separator: "/",
tmpdir: "/tmp",
cwd: "/",
driveCwds: {},
pid: process.pid,
});
} else {
send(3, id, { code: "EINVAL", message: `unknown operation ${json.op}` });
}
}
});
process.stdin.on("end", () => process.exit(0));

View File

@@ -0,0 +1,176 @@
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { BACKGROUND_CONTEXT, withAbortSignal } from "@earendil-works/chord/context";
import { getOrThrow, type ShellOutputInfo } from "@earendil-works/pi-durable/env";
import { afterAll, describe, expect, it } from "vitest";
import { Connection, nodePlatform } from "../src/connection.ts";
import { RemoteExecutionEnv } from "../src/remote-env.ts";
import { daemon } from "./daemon.ts";
const context = BACKGROUND_CONTEXT;
const connections: Connection[] = [];
const dirs: string[] = [];
afterAll(() => {
for (const connection of connections) connection.close();
for (const dir of dirs) rmSync(dir, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 });
});
function environment(): { env: RemoteExecutionEnv; connection: Connection } {
const cwd = mkdtempSync(join(tmpdir(), "pi-env-remote-"));
dirs.push(cwd);
const connection = new Connection({ command: [daemon] });
connections.push(connection);
return { env: new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd }), connection };
}
describe("RemoteExecutionEnv", () => {
it("transfers only about a window of output a caller keeps, with exact counts", async () => {
const { env } = environment();
const total = 5_000_000;
let delivered = 0;
let skipped = 0;
let tail = "";
const result = await env.exec(
["sh", "-c", `yes 0123456789abcdef | head -c ${total}`],
{
window: { maxBytes: 1000, maxLines: 20, minIntervalMs: 20, bytesPerSecond: 1_000_000 },
onOutput: (text: string, _context, info: ShellOutputInfo) => {
delivered += Buffer.byteLength(text);
if (info.skipped) {
skipped += info.skipped.bytes;
tail = "";
}
tail += text;
},
},
context,
);
expect(getOrThrow(result).exitCode).toBe(0);
expect(delivered + skipped).toBe(total);
expect(skipped).toBeGreaterThan(0);
expect(delivered).toBeLessThan(total / 10);
// After the last skip comes everything to the end, more than the window.
const full = "0123456789abcdef\n".repeat(Math.ceil(total / 17)).slice(0, total);
expect(full.endsWith(tail)).toBe(true);
expect(tail.length > 1000 || tail.split("\n").length - 1 > 20).toBe(true);
}, 60_000);
it("aborts commands cancelled right after they are sent", async () => {
const { env } = environment();
const started = Date.now();
for (let attempt = 0; attempt < 20; attempt++) {
const controller = new AbortController();
const running = env.exec(["sleep", "5"], undefined, withAbortSignal(controller.signal, context));
if (attempt % 2 === 0) controller.abort();
else setTimeout(() => controller.abort(), attempt % 5);
const result = await running;
expect(result.ok ? "ok" : result.error.code).toBe("aborted");
}
expect(Date.now() - started).toBeLessThan(10_000);
}, 30_000);
it("reports the remote system in Node's names", async () => {
const { connection } = environment();
const { platform, arch } = nodePlatform(await connection.info());
expect(platform).toBe(process.platform);
// CI runs the darwin-x64 release daemon on arm64 macOS through Rosetta, so it reports x64 to arm64 Node.
const rosetta = process.platform === "darwin" && process.arch === "arm64" && arch === "x64";
if (!rosetta) expect(arch).toBe(process.arch);
expect(nodePlatform({ os: "windows", arch: "aarch64" })).toEqual({ platform: "win32", arch: "arm64" });
expect(nodePlatform({ os: "macos", arch: "x86_64" })).toEqual({ platform: "darwin", arch: "x64" });
});
it("reports listFiles as not supported by daemons released before it", async () => {
const { env } = environment();
expect((await env.connection.info()).features).toContain("listFiles");
const connection = new Connection({ command: [process.execPath, join(import.meta.dirname, "old-daemon.ts")] });
connections.push(connection);
const old = new RemoteExecutionEnv({ connection, id: "pi-env:old", cwd: env.cwd });
const listed = await old.listFiles(".", undefined, context);
expect(listed.ok ? "ok" : listed.error.code).toBe("not_supported");
});
it("cancels a listing in the daemon", async () => {
const { env, connection } = environment();
for (let dir = 0; dir < 100; dir++) {
mkdirSync(join(env.cwd, `d${dir}`));
for (let file = 0; file < 100; file++) writeFileSync(join(env.cwd, `d${dir}`, `f${file}`), "");
}
// The cancel follows the request at once, bypassing the client's own abort check, so the daemon stops the walk.
const controller = new AbortController();
controller.abort();
const cancelled = connection.request("listFiles", { path: env.cwd }, { signal: controller.signal });
await expect(cancelled).rejects.toMatchObject({ code: "aborted" });
const listed = getOrThrow(await env.listFiles(".", undefined, context));
expect([listed.entries.length, listed.truncated]).toEqual([10_100, false]);
const bounded = getOrThrow(await env.listFiles(".", { maxEntries: 5000 }, context));
expect([bounded.entries.length, bounded.truncated]).toEqual([5000, true]);
// Creating 10,000 files takes several seconds on Windows CI.
}, 30_000);
it("times out a command that produces no output", async () => {
const { env } = environment();
const started = Date.now();
const result = await env.exec(["sleep", "5"], { timeout: 0.3 }, context);
expect(result.ok ? "ok" : result.error.code).toBe("timeout");
expect(Date.now() - started).toBeLessThan(3000);
});
it("fails requests on handles of a lost connection instead of reaching the new daemon", async () => {
const { env, connection } = environment();
getOrThrow(await env.writeFile("a.txt", "AAA", context));
getOrThrow(await env.writeFile("b.txt", "BBB", context));
const a = getOrThrow(await env.openBinaryReader("a.txt", undefined, context));
const { pid } = await connection.info();
process.kill(pid, "SIGKILL");
await new Promise((done) => setTimeout(done, 300));
// The new daemon numbers its handles from 1 again.
const b = getOrThrow(await env.openBinaryReader("b.txt", undefined, context));
expect((await connection.info()).pid).not.toBe(pid);
const stale = await a.read(0, 3, context);
expect(stale.ok ? new TextDecoder().decode(stale.value) : stale.error.code).toBe("unknown");
expect(new TextDecoder().decode(getOrThrow(await b.read(0, 3, context)))).toBe("BBB");
await a.close(context);
await b.close(context);
});
it("computes the command at each start and retries a start that failed", async () => {
const cwd = mkdtempSync(join(tmpdir(), "pi-env-remote-"));
dirs.push(cwd);
let starts = 0;
const connection = new Connection({
command: async () => {
starts++;
if (starts === 1) throw new Error("ssh gpu-box failed: Network is unreachable");
return [daemon];
},
});
connections.push(connection);
const env = new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd });
const offline = await env.readTextFile("missing.txt", context);
expect(offline.ok ? "ok" : [offline.error.code, offline.error.message]).toEqual([
"unknown",
"ssh gpu-box failed: Network is unreachable",
]);
const command = await env.exec(["sh", "-c", "exit 0"], undefined, context);
expect(command.ok ? command.value.exitCode : command.error.code).toBe(0);
expect(starts).toBe(2);
});
it("writes and reads files larger than one transfer chunk in order", async () => {
const { env } = environment();
const content = new Uint8Array(3_500_017);
for (let index = 0; index < content.length; index++) content[index] = (index * 7919) % 251;
getOrThrow(await env.writeFile("big.bin", content, context));
getOrThrow(await env.appendFile("big.bin", content, context));
const read = getOrThrow(await env.readBinaryFile("big.bin", context));
expect(read.length).toBe(content.length * 2);
expect(Buffer.compare(Buffer.from(read.subarray(0, content.length)), Buffer.from(content))).toBe(0);
expect(Buffer.compare(Buffer.from(read.subarray(content.length)), Buffer.from(content))).toBe(0);
const reader = getOrThrow(await env.openBinaryReader("big.bin", undefined, context));
const range = getOrThrow(await reader.read(1_000_003, 2_000_000, context));
await reader.close(context);
expect(Buffer.compare(Buffer.from(range), Buffer.from(read.subarray(1_000_003, 3_000_003)))).toBe(0);
}, 60_000);
});

View File

@@ -0,0 +1,92 @@
import { mkdtempSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { getOrThrow } from "@earendil-works/pi-durable/env";
import { registerEnvConformance } from "@earendil-works/pi-durable/testing";
import { afterAll, describe, expect, it } from "vitest";
import { type Connection, nodePlatform } from "../src/connection.ts";
import { RemoteExecutionEnv } from "../src/remote-env.ts";
import {
acceptHostKey,
connectSsh,
deployDaemon,
detectPlatform,
type RemotePlatform,
type SshTarget,
scanHostKey,
} from "../src/ssh.ts";
import { daemon } from "./daemon.ts";
/**
* The conformance suite over a real SSH server on this machine, configured by CI: `PI_ENV_SSH_HOST`, `PI_ENV_SSH_USER`
* `PI_ENV_SSH_KEY` and `PI_ENV_SSH_PROGRAM`. On Windows this covers deployment and the daemon's launch through the server's default shell.
*/
const host = process.env.PI_ENV_SSH_HOST;
const root = mkdtempSync(join(tmpdir(), "pi-env-ssh-external-"));
let connected: Promise<{ connection: Connection; remote: RemotePlatform }> | undefined;
async function connect(): Promise<{ connection: Connection; remote: RemotePlatform }> {
connected ??= (async () => {
const target: SshTarget = {
host: host!,
...(process.env.PI_ENV_SSH_USER === undefined ? {} : { user: process.env.PI_ENV_SSH_USER }),
...(process.env.PI_ENV_SSH_PORT === undefined ? {} : { port: Number(process.env.PI_ENV_SSH_PORT) }),
...(process.env.PI_ENV_SSH_KEY === undefined ? {} : { identityFile: process.env.PI_ENV_SSH_KEY }),
...(process.env.PI_ENV_SSH_PROGRAM === undefined ? {} : { ssh: process.env.PI_ENV_SSH_PROGRAM }),
knownHostsFile: join(root, "known_hosts"),
hostKeyAlias: "pi-env-external",
};
// Each stage is logged, so a hang in CI shows where it is.
const log = (text: string) => process.stderr.write(`[ssh-external ${new Date().toISOString()}] ${text}\n`);
log("scanning the host key");
await acceptHostKey(target, (await scanHostKey(target)).lines);
log("detecting the platform");
const remote = await detectPlatform(target);
log(`detected ${JSON.stringify(remote)}; deploying`);
log(`deployed ${await deployDaemon(target, remote, daemon)}; connecting`);
const result = await connectSsh({ ...target, binary: daemon, onLog: (text) => log(`daemon: ${text.trim()}`) });
log("connected; saying hello");
log(`hello ${JSON.stringify(await result.connection.info())}`);
return result;
})();
return connected;
}
afterAll(async () => {
if (connected) (await connected).connection.close();
rmSync(root, { recursive: true, force: true, maxRetries: 50, retryDelay: 100 });
});
const windows = process.platform === "win32";
const gitBash = join(process.env.ProgramFiles ?? "C:\\Program Files", "Git", "bin", "bash.exe");
describe.skipIf(host === undefined)("SSH to this machine's server", () => {
it("reaches the remote system through its default shell", async () => {
const { connection, remote } = await connect();
const info = await connection.info();
expect(nodePlatform(info).platform).toBe(process.platform);
if (windows) expect(remote.shell).toBe(process.env.PI_ENV_SSH_SHELL ?? "cmd");
}, 120_000);
});
if (host !== undefined) {
registerEnvConformance(
{ describe, expect, it },
"RemoteExecutionEnv over SSH",
async (use) => {
const { connection } = await connect();
const info = await connection.info();
const home = new RemoteExecutionEnv({ connection, id: "pi-env:external", cwd: info.home });
const cwd = getOrThrow(await home.createTempDir("pi-env-conformance-", BACKGROUND_CONTEXT));
try {
await use(
new RemoteExecutionEnv({ connection, id: "pi-env:external", cwd, watch: { pollIntervalMs: 100 } }),
);
} finally {
await home.remove(cwd, { recursive: true, force: true }, BACKGROUND_CONTEXT);
}
},
windows ? { shell: [gitBash, "-c"], symlinks: false, listFiles: true } : { listFiles: true },
);
}

View File

@@ -0,0 +1,299 @@
import { type ChildProcess, execFileSync, spawn } from "node:child_process";
import { createHash } from "node:crypto";
import {
existsSync,
mkdirSync,
mkdtempSync,
readdirSync,
readFileSync,
rmSync,
statSync,
writeFileSync,
} from "node:fs";
import { createServer, Socket } from "node:net";
import { tmpdir, userInfo } from "node:os";
import { join } from "node:path";
import { BACKGROUND_CONTEXT } from "@earendil-works/chord/context";
import { getOrThrow } from "@earendil-works/pi-durable/env";
import { afterAll, beforeAll, describe, expect, it } from "vitest";
import { Connection } from "../src/connection.ts";
import { LocalExecutionEnv } from "../src/list-files.ts";
import { RemoteExecutionEnv } from "../src/remote-env.ts";
import {
acceptHostKey,
connectSsh,
forgetHostKey,
HostKeyChangedError,
HostKeyUnknownError,
type SshTarget,
scanHostKey,
sshConnection,
} from "../src/ssh.ts";
import { daemon } from "./daemon.ts";
/** The deployed daemon is named by its content. */
const deployedName = `pi-env-${createHash("sha256").update(readFileSync(daemon)).digest("hex").slice(0, 32)}`;
const context = BACKGROUND_CONTEXT;
function which(program: string): string | undefined {
try {
return execFileSync("sh", ["-c", `command -v ${program}`], { encoding: "utf8" }).trim() || undefined;
} catch {
return undefined;
}
}
const sshd =
process.platform === "win32"
? undefined
: (which("sshd") ?? (existsSync("/usr/sbin/sshd") ? "/usr/sbin/sshd" : undefined));
async function freePort(): Promise<number> {
return new Promise((resolve) => {
const server = createServer().listen(0, "127.0.0.1", () => {
const { port } = server.address() as { port: number };
server.close(() => resolve(port));
});
});
}
async function waitForPort(port: number): Promise<void> {
for (let attempt = 0; attempt < 100; attempt++) {
const open = await new Promise<boolean>((resolve) => {
const socket = new Socket();
socket.once("connect", () => resolve(true)).once("error", () => resolve(false));
socket.connect(port, "127.0.0.1");
}).finally(() => undefined);
if (open) return;
await new Promise((resolve) => setTimeout(resolve, 50));
}
throw new Error("sshd did not start");
}
// A disposable sshd on localhost with its own host key, client key and home directory.
describe.skipIf(sshd === undefined)("SSH bootstrap", () => {
const root = mkdtempSync(join(tmpdir(), "pi-env-ssh-"));
const home = join(root, "home");
let server: ChildProcess | undefined;
let target: SshTarget;
beforeAll(async () => {
execFileSync("mkdir", ["-p", home]);
execFileSync("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", join(root, "host_key")]);
execFileSync("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", join(root, "client_key")]);
writeFileSync(join(root, "authorized_keys"), readFileSync(join(root, "client_key.pub")));
const port = await freePort();
writeFileSync(
join(root, "sshd_config"),
[
`Port ${port}`,
"ListenAddress 127.0.0.1",
`HostKey ${join(root, "host_key")}`,
`AuthorizedKeysFile ${join(root, "authorized_keys")}`,
`PidFile ${join(root, "sshd.pid")}`,
"PasswordAuthentication no",
"KbdInteractiveAuthentication no",
"StrictModes no",
`SetEnv HOME=${home}`,
"",
].join("\n"),
);
writeFileSync(join(root, "ssh_config"), "");
server = spawn(sshd!, ["-D", "-e", "-f", join(root, "sshd_config")], { stdio: "ignore" });
await waitForPort(port);
target = {
host: "127.0.0.1",
port,
user: userInfo().username,
identityFile: join(root, "client_key"),
knownHostsFile: join(root, "known_hosts"),
hostKeyAlias: "pi-env-test",
configFile: join(root, "ssh_config"),
};
});
afterAll(() => {
server?.kill();
rmSync(root, { recursive: true, force: true });
});
it("refuses an untrusted host, then deploys and runs the daemon once its key is accepted", async () => {
await expect(connectSsh({ ...target, binary: daemon })).rejects.toBeInstanceOf(HostKeyUnknownError);
const scanned = await scanHostKey(target);
const expected = execFileSync("ssh-keygen", ["-lf", join(root, "host_key.pub")], { encoding: "utf8" }).split(
" ",
)[1];
expect(scanned.fingerprints.join("\n")).toContain(expected);
await acceptHostKey(target, scanned.lines);
mkdirSync(join(home, ".pi/mobile/tools"), { recursive: true });
writeFileSync(join(home, ".pi/mobile/tools/pi-env-0123456789abcdef0123456789abcdef"), "old");
const { connection, remote } = await connectSsh({ ...target, binary: daemon });
try {
expect(remote.home).toBe(home);
const deployed = readFileSync(join(home, ".pi/mobile/tools", deployedName));
expect(deployed.equals(readFileSync(daemon))).toBe(true);
// Daemons of other contents are removed.
expect(readdirSync(join(home, ".pi/mobile/tools")).filter((name) => name.startsWith("pi-env-"))).toEqual([
deployedName,
]);
const env = new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd: home });
getOrThrow(await env.writeFile("over-ssh.txt", "hello", context));
expect(getOrThrow(await env.readTextFile("over-ssh.txt", context))).toBe("hello");
const output: string[] = [];
const result = getOrThrow(
await env.exec(["sh", "-c", 'printf "%s" "$HOME"'], { onOutput: (text) => output.push(text) }, context),
);
expect(result.exitCode).toBe(0);
expect(output.join("")).toBe(home);
} finally {
connection.close();
}
}, 60_000);
it("reuses a verified daemon and replaces a tampered one", async () => {
const file = join(home, ".pi/mobile/tools", deployedName);
const before = statSync(file).mtimeMs;
(await connectSsh({ ...target, binary: daemon })).connection.close();
expect(statSync(file).mtimeMs).toBe(before);
writeFileSync(file, "tampered");
const { connection } = await connectSsh({ ...target, binary: daemon });
try {
expect(readFileSync(file).equals(readFileSync(daemon))).toBe(true);
expect((await connection.info()).home).toBe(home);
} finally {
connection.close();
}
}, 60_000);
it("verifies the daemon again before starting it after a lost connection", async () => {
const { connection } = await connectSsh({ ...target, binary: daemon });
const file = join(home, ".pi/mobile/tools", deployedName);
try {
const { pid } = await connection.info();
process.kill(pid, "SIGKILL");
await new Promise((done) => setTimeout(done, 300));
// Linux refuses to write a running program's file.
writeFileSync(file, "tampered");
// The next start redeploys the verified binary instead of running whatever is there.
expect((await connection.info()).pid).not.toBe(pid);
expect(readFileSync(file).equals(readFileSync(daemon))).toBe(true);
} finally {
connection.close();
}
}, 60_000);
it("lists files over SSH as the local environment lists them", async () => {
const tree = join(home, "tree");
for (const file of ["a/b.txt", "a/c/d.txt", ".hidden", "out/x.js", "skip/y.txt", ".git/HEAD"]) {
mkdirSync(join(tree, file, ".."), { recursive: true });
writeFileSync(join(tree, file), "");
}
mkdirSync(join(tree, "empty"));
writeFileSync(join(tree, ".gitignore"), "out/\n");
const { connection } = await connectSsh({ ...target, binary: daemon });
const local = new Connection({ command: [daemon] });
try {
const remote = new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd: home });
const listing = getOrThrow(await remote.listFiles("tree", { exclude: ["skip"] }, context));
expect(listing).toEqual({
entries: [
{ path: ".gitignore", kind: "file" },
{ path: ".hidden", kind: "file" },
{ path: "a", kind: "directory" },
{ path: "a/b.txt", kind: "file" },
{ path: "a/c", kind: "directory" },
{ path: "a/c/d.txt", kind: "file" },
{ path: "empty", kind: "directory" },
],
truncated: false,
});
const env = new LocalExecutionEnv({ connection: local, cwd: home });
expect(getOrThrow(await env.listFiles("tree", { exclude: ["skip"] }, context))).toEqual(listing);
} finally {
connection.close();
local.close();
}
}, 60_000);
it("starts the daemon through the login shell only when asked", async () => {
// The login shell is the account's shell: sh and bash read .profile, zsh (the macOS default) reads .zprofile.
writeFileSync(join(home, ".profile"), "export PI_ENV_LOGIN=yes\n");
writeFileSync(join(home, ".zprofile"), "export PI_ENV_LOGIN=yes\n");
const login = async (loginShell: boolean) => {
const { connection } = await connectSsh({ ...target, binary: daemon, loginShell });
try {
const env = new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd: home });
const output: string[] = [];
getOrThrow(
await env.exec(
["sh", "-c", 'printf "%s" "$PI_ENV_LOGIN"'],
{ onOutput: (text) => output.push(text) },
context,
),
);
return output.join("");
} finally {
connection.close();
}
};
expect(await login(false)).toBe("");
expect(await login(true)).toBe("yes");
}, 60_000);
it("connects lazily and reports a failed start as the error of the operation", async () => {
const lazyTarget = { ...target, knownHostsFile: join(root, "lazy_known_hosts") };
const { connection, remote } = sshConnection({ ...lazyTarget, binary: daemon });
try {
expect(remote()).toBeUndefined();
const env = new RemoteExecutionEnv({ connection, id: "pi-env:test", cwd: home });
// Nothing is trusted yet: each operation fails with the reason and tries again.
const read = await env.readTextFile("missing.txt", context);
expect(read.ok ? "ok" : read.error.message).toMatch(/not trusted/);
const command = await env.exec(["sh", "-c", "exit 0"], undefined, context);
expect(command.ok ? "ok" : command.error.code).toBe("spawn_error");
await acceptHostKey(lazyTarget, (await scanHostKey(lazyTarget)).lines);
const accepted = await env.exec(["sh", "-c", "exit 0"], undefined, context);
expect(accepted.ok ? accepted.value.exitCode : accepted.error.message).toBe(0);
expect(remote()?.home).toBe(home);
} finally {
connection.close();
}
}, 60_000);
it("accepts only host keys for the alias and never replaces a trusted key silently", async () => {
const knownHosts = join(root, "accept_known_hosts");
const scoped = { ...target, knownHostsFile: knownHosts };
const key = (name: string) => {
const file = join(root, name);
execFileSync("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", file]);
return readFileSync(`${file}.pub`, "utf8").split(" ").slice(0, 2).join(" ");
};
const first = key("accept_first");
const second = key("accept_second");
await expect(acceptHostKey(scoped, [`other-alias ${first}`])).rejects.toThrow();
await expect(acceptHostKey(scoped, [`@cert-authority pi-env-test ${first}`])).rejects.toThrow();
await Promise.all([
acceptHostKey(scoped, [`pi-env-test ${first}`]),
acceptHostKey(scoped, [`pi-env-test ${first}`]),
]);
expect(readFileSync(knownHosts, "utf8")).toBe(`pi-env-test ${first}\n`);
await expect(acceptHostKey(scoped, [`pi-env-test ${second}`])).rejects.toBeInstanceOf(HostKeyChangedError);
await forgetHostKey(scoped);
await acceptHostKey(scoped, [`pi-env-test ${second}`]);
expect(readFileSync(knownHosts, "utf8")).toBe(`pi-env-test ${second}\n`);
});
it("refuses a changed host key", async () => {
const other = join(root, "other_key");
execFileSync("ssh-keygen", ["-q", "-t", "ed25519", "-N", "", "-f", other]);
const key = readFileSync(`${other}.pub`, "utf8").split(" ").slice(0, 2).join(" ");
const knownHosts = join(root, "changed_known_hosts");
writeFileSync(knownHosts, `pi-env-test ${key}\n`);
await expect(connectSsh({ ...target, knownHostsFile: knownHosts, binary: daemon })).rejects.toBeInstanceOf(
HostKeyChangedError,
);
}, 60_000);
});

View File

@@ -0,0 +1,162 @@
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterAll, describe, expect, it } from "vitest";
import { Connection, RemoteError } from "../src/connection.ts";
import { openTerminal } from "../src/terminal.ts";
import { daemon } from "./daemon.ts";
const connections: Connection[] = [];
const dirs: string[] = [];
afterAll(() => {
for (const connection of connections) connection.close();
for (const dir of dirs) rmSync(dir, { recursive: true, force: true });
});
function connect(): { connection: Connection; cwd: string } {
const cwd = mkdtempSync(join(tmpdir(), "pi-env-terminal-"));
dirs.push(cwd);
const connection = new Connection({ command: [daemon] });
connections.push(connection);
return { connection, cwd };
}
/** Collects output and waits for text to appear in it. */
function collector(): {
onData: (bytes: Uint8Array) => void;
bytes: () => Buffer;
waitFor: (text: string) => Promise<void>;
} {
const chunks: Buffer[] = [];
const waiting: { text: string; resolve: () => void }[] = [];
const bytes = () => Buffer.concat(chunks);
return {
bytes,
onData: (data) => {
chunks.push(Buffer.from(data));
const all = bytes().toString("latin1");
for (const wait of waiting.splice(0)) {
if (all.includes(wait.text)) wait.resolve();
else waiting.push(wait);
}
},
waitFor: (text) =>
bytes().toString("latin1").includes(text)
? Promise.resolve()
: new Promise((resolve) => waiting.push({ text, resolve })),
};
}
describe.skipIf(process.platform === "win32")("openTerminal", () => {
it("runs a program in a terminal with input, resizing and its exit code", async () => {
const { connection, cwd } = connect();
const output = collector();
const terminal = await openTerminal(connection, {
cwd,
argv: ["/bin/sh"],
env: { PS1: "$ ", TERM: "xterm-256color" },
cols: 80,
rows: 24,
onData: output.onData,
});
expect(terminal.pid).toBeGreaterThan(0);
terminal.write("test -t 0 && echo IS-A-TTY; stty size; echo $TERM\n");
await output.waitFor("IS-A-TTY");
await output.waitFor("24 80");
await output.waitFor("xterm-256color");
terminal.resize(100, 30);
terminal.write("stty size\n");
await output.waitFor("30 100");
terminal.write("exit 3\n");
expect(await terminal.exited).toBe(3);
}, 20_000);
it("reports the job in the foreground, not the shell at its prompt", async () => {
const { connection, cwd } = connect();
const output = collector();
const terminal = await openTerminal(connection, {
cwd,
// Interactive, for job control: the shell puts each job in front of the terminal.
argv: ["/bin/sh", "-i"],
env: { PS1: "$ ", TERM: "xterm-256color" },
cols: 80,
rows: 24,
onData: output.onData,
});
terminal.write("echo READY\n");
await output.waitFor("READY");
expect(await terminal.foreground()).toBeUndefined();
terminal.write("sleep 100\n");
// The shell forks before it execs: wait until the foreground job is the command itself, not the forked shell.
const named = process.platform === "darwin" || process.platform === "linux";
let foreground = await terminal.foreground();
for (let tries = 0; (!foreground || (named && foreground.name !== "sleep")) && tries < 100; tries++) {
await new Promise((resolve) => setTimeout(resolve, 20));
foreground = await terminal.foreground();
}
expect(foreground?.pid).toBeGreaterThan(0);
expect(foreground?.pid).not.toBe(terminal.pid);
if (named) expect(foreground?.name).toBe("sleep");
terminal.close();
await expect(terminal.exited).rejects.toMatchObject({ code: "aborted" });
await expect(terminal.foreground()).rejects.toMatchObject({ code: "EBADF" });
}, 20_000);
it("delivers raw bytes, not decoded text", async () => {
const { connection, cwd } = connect();
const output = collector();
const terminal = await openTerminal(connection, {
cwd,
argv: ["sh", "-c", "printf 'a\\377b'"],
cols: 80,
rows: 24,
onData: output.onData,
});
expect(await terminal.exited).toBe(0);
expect([...output.bytes()]).toEqual([0x61, 0xff, 0x62]);
});
it("keeps input in order", async () => {
const { connection, cwd } = connect();
const terminal = await openTerminal(connection, {
cwd,
argv: ["sh", "-c", "head -c 40000 > input.txt"],
cols: 80,
rows: 24,
onData: () => {},
});
let text = "";
for (let index = 0; index < 4000; index++) {
const piece = `${String(index).padStart(9, "0")}\n`;
text += piece;
terminal.write(piece);
}
expect(await terminal.exited).toBe(0);
expect(readFileSync(join(cwd, "input.txt"), "utf8")).toBe(text);
}, 20_000);
it("kills the program on close", async () => {
const { connection, cwd } = connect();
const terminal = await openTerminal(connection, {
cwd,
argv: ["sleep", "100"],
cols: 80,
rows: 24,
onData: () => {},
});
terminal.close();
await expect(terminal.exited).rejects.toMatchObject({ code: "aborted" });
});
it("fails to open in a missing directory", async () => {
const { connection, cwd } = connect();
await expect(
openTerminal(connection, { cwd: join(cwd, "missing"), cols: 80, rows: 24, onData: () => {} }),
).rejects.toBeInstanceOf(RemoteError);
});
it("rejects writes to a terminal that does not exist", async () => {
const { connection } = connect();
await expect(connection.request("ptyWrite", { target: 999 })).rejects.toMatchObject({ code: "EBADF" });
});
});