chore(vendor): import pi 0.85.1
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: d981de1229ef899957bbe968bc8dcda02a21f477
This commit is contained in:
1610
third_party/pi/packages/agent/test/agent-loop.test.ts
vendored
Normal file
1610
third_party/pi/packages/agent/test/agent-loop.test.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
810
third_party/pi/packages/agent/test/agent.test.ts
vendored
Normal file
810
third_party/pi/packages/agent/test/agent.test.ts
vendored
Normal file
@@ -0,0 +1,810 @@
|
||||
import { type AssistantMessage, type AssistantMessageEvent, EventStream, getModel } from "@earendil-works/pi-ai/compat";
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
Agent,
|
||||
type AgentEvent,
|
||||
type AgentTool,
|
||||
type AgentToolUpdateCallback,
|
||||
type StreamFn,
|
||||
setDefaultStreamFn,
|
||||
} from "../src/index.ts";
|
||||
|
||||
// Mock stream that mimics AssistantMessageEventStream
|
||||
class MockAssistantStream extends EventStream<AssistantMessageEvent, AssistantMessage> {
|
||||
constructor() {
|
||||
super(
|
||||
(event) => event.type === "done" || event.type === "error",
|
||||
(event) => {
|
||||
if (event.type === "done") return event.message;
|
||||
if (event.type === "error") return event.error;
|
||||
throw new Error("Unexpected event type");
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function createAssistantMessage(text: string): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text }],
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
model: "mock",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
type ToolCallContent = Extract<AssistantMessage["content"][number], { type: "toolCall" }>;
|
||||
|
||||
function createAssistantToolUseMessage(content: ToolCallContent[]): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content,
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
model: "mock",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "toolUse",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
const unusedStreamFunction: StreamFn = () => {
|
||||
throw new Error("Unexpected stream call");
|
||||
};
|
||||
|
||||
function createDeferred(): {
|
||||
promise: Promise<void>;
|
||||
resolve: () => void;
|
||||
} {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe("Agent", () => {
|
||||
it("uses the configured default when a legacy caller omits streamFn", async () => {
|
||||
let calls = 0;
|
||||
setDefaultStreamFn(() => {
|
||||
calls++;
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
const message = createAssistantMessage("fallback");
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
});
|
||||
return stream;
|
||||
});
|
||||
|
||||
try {
|
||||
const agent = Reflect.construct(Agent, [{}]) as Agent;
|
||||
await agent.prompt("Hello");
|
||||
expect(calls).toBe(1);
|
||||
} finally {
|
||||
setDefaultStreamFn(undefined);
|
||||
}
|
||||
});
|
||||
|
||||
it("should create an agent instance with default state", () => {
|
||||
const agent = new Agent({ streamFn: unusedStreamFunction });
|
||||
|
||||
expect(agent.state).toBeDefined();
|
||||
expect(agent.state.systemPrompt).toBe("");
|
||||
expect(agent.state.model).toBeDefined();
|
||||
expect(agent.state.thinkingLevel).toBe("off");
|
||||
expect(agent.state.tools).toEqual([]);
|
||||
expect(agent.state.messages).toEqual([]);
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.streamingMessage).toBe(undefined);
|
||||
expect(agent.state.pendingToolCalls).toEqual(new Set());
|
||||
expect(agent.state.errorMessage).toBeUndefined();
|
||||
});
|
||||
|
||||
it("should create an agent instance with custom initial state", () => {
|
||||
const customModel = getModel("openai", "gpt-4o-mini");
|
||||
const agent = new Agent({
|
||||
streamFn: unusedStreamFunction,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
model: customModel,
|
||||
thinkingLevel: "low",
|
||||
},
|
||||
});
|
||||
|
||||
expect(agent.state.systemPrompt).toBe("You are a helpful assistant.");
|
||||
expect(agent.state.model).toBe(customModel);
|
||||
expect(agent.state.thinkingLevel).toBe("low");
|
||||
});
|
||||
|
||||
it("should subscribe to events", () => {
|
||||
const agent = new Agent({ streamFn: unusedStreamFunction });
|
||||
|
||||
let eventCount = 0;
|
||||
const unsubscribe = agent.subscribe((_event) => {
|
||||
eventCount++;
|
||||
});
|
||||
|
||||
// No initial event on subscribe
|
||||
expect(eventCount).toBe(0);
|
||||
|
||||
// State mutators don't emit events
|
||||
agent.state.systemPrompt = "Test prompt";
|
||||
expect(eventCount).toBe(0);
|
||||
expect(agent.state.systemPrompt).toBe("Test prompt");
|
||||
|
||||
// Unsubscribe should work
|
||||
unsubscribe();
|
||||
agent.state.systemPrompt = "Another prompt";
|
||||
expect(eventCount).toBe(0); // Should not increase
|
||||
});
|
||||
|
||||
it("emits full lifecycle events for thrown run failures", async () => {
|
||||
const agent = new Agent({
|
||||
streamFn: () => {
|
||||
throw new Error("provider exploded");
|
||||
},
|
||||
});
|
||||
const events: string[] = [];
|
||||
agent.subscribe((event) => {
|
||||
events.push(event.type);
|
||||
});
|
||||
|
||||
await agent.prompt("hello");
|
||||
|
||||
expect(events).toEqual([
|
||||
"agent_start",
|
||||
"turn_start",
|
||||
"message_start",
|
||||
"message_end",
|
||||
"message_start",
|
||||
"message_end",
|
||||
"turn_end",
|
||||
"agent_end",
|
||||
]);
|
||||
const lastMessage = agent.state.messages[agent.state.messages.length - 1];
|
||||
expect(lastMessage?.role).toBe("assistant");
|
||||
if (lastMessage?.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(lastMessage.stopReason).toBe("error");
|
||||
expect(lastMessage.errorMessage).toBe("provider exploded");
|
||||
expect(agent.state.errorMessage).toBe("provider exploded");
|
||||
});
|
||||
|
||||
it("should await async subscribers before prompt resolves", async () => {
|
||||
const barrier = createDeferred();
|
||||
const agent = new Agent({
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "done", reason: "stop", message: createAssistantMessage("ok") });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
let listenerFinished = false;
|
||||
agent.subscribe(async (event) => {
|
||||
if (event.type === "agent_end") {
|
||||
await barrier.promise;
|
||||
listenerFinished = true;
|
||||
}
|
||||
});
|
||||
|
||||
let promptResolved = false;
|
||||
const promptPromise = agent.prompt("hello").then(() => {
|
||||
promptResolved = true;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(promptResolved).toBe(false);
|
||||
expect(listenerFinished).toBe(false);
|
||||
expect(agent.state.isStreaming).toBe(true);
|
||||
|
||||
barrier.resolve();
|
||||
await promptPromise;
|
||||
|
||||
expect(listenerFinished).toBe(true);
|
||||
expect(promptResolved).toBe(true);
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
});
|
||||
|
||||
it("waitForIdle should wait for async subscribers", async () => {
|
||||
const barrier = createDeferred();
|
||||
const agent = new Agent({
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "done", reason: "stop", message: createAssistantMessage("ok") });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
agent.subscribe(async (event) => {
|
||||
if (event.type === "message_end" && event.message.role === "assistant") {
|
||||
await barrier.promise;
|
||||
}
|
||||
});
|
||||
|
||||
const promptPromise = agent.prompt("hello");
|
||||
let idleResolved = false;
|
||||
const idlePromise = agent.waitForIdle().then(() => {
|
||||
idleResolved = true;
|
||||
});
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(idleResolved).toBe(false);
|
||||
expect(agent.state.isStreaming).toBe(true);
|
||||
|
||||
barrier.resolve();
|
||||
await Promise.all([promptPromise, idlePromise]);
|
||||
|
||||
expect(idleResolved).toBe(true);
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
});
|
||||
|
||||
it("should pass the active abort signal to subscribers", async () => {
|
||||
let receivedSignal: AbortSignal | undefined;
|
||||
const agent = new Agent({
|
||||
streamFn: (_model, _context, options) => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "start", partial: createAssistantMessage("") });
|
||||
const checkAbort = () => {
|
||||
if (options?.signal?.aborted) {
|
||||
stream.push({ type: "error", reason: "aborted", error: createAssistantMessage("Aborted") });
|
||||
} else {
|
||||
setTimeout(checkAbort, 5);
|
||||
}
|
||||
};
|
||||
checkAbort();
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
agent.subscribe((event, signal) => {
|
||||
if (event.type === "agent_start") {
|
||||
receivedSignal = signal;
|
||||
}
|
||||
});
|
||||
|
||||
const promptPromise = agent.prompt("hello");
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
|
||||
expect(receivedSignal).toBeDefined();
|
||||
expect(receivedSignal?.aborted).toBe(false);
|
||||
|
||||
agent.abort();
|
||||
await promptPromise;
|
||||
|
||||
expect(receivedSignal?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("should ignore tool updates after the tool execution settles", async () => {
|
||||
const toolSchema = Type.Object({});
|
||||
let delayedUpdate: AgentToolUpdateCallback<{ status: string }> | undefined;
|
||||
const events: AgentEvent[] = [];
|
||||
const unhandledRejections: unknown[] = [];
|
||||
const onUnhandledRejection = (error: unknown) => {
|
||||
unhandledRejections.push(error);
|
||||
};
|
||||
const tool: AgentTool<typeof toolSchema, { status: string }> = {
|
||||
name: "delayed_tool",
|
||||
label: "Delayed Tool",
|
||||
description: "Captures progress callbacks",
|
||||
parameters: toolSchema,
|
||||
async execute(_toolCallId, _params, _signal, onUpdate) {
|
||||
delayedUpdate = onUpdate;
|
||||
onUpdate?.({
|
||||
content: [{ type: "text", text: "running" }],
|
||||
details: { status: "running" },
|
||||
});
|
||||
return {
|
||||
content: [{ type: "text", text: "ok" }],
|
||||
details: { status: "done" },
|
||||
terminate: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const agent = new Agent({
|
||||
initialState: { tools: [tool] },
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({
|
||||
type: "done",
|
||||
reason: "toolUse",
|
||||
message: createAssistantToolUseMessage([
|
||||
{ type: "toolCall", id: "call-1", name: "delayed_tool", arguments: {} },
|
||||
]),
|
||||
});
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
agent.subscribe((event) => {
|
||||
events.push(event);
|
||||
});
|
||||
|
||||
process.on("unhandledRejection", onUnhandledRejection);
|
||||
try {
|
||||
await agent.prompt("run tool");
|
||||
const eventCountAfterPrompt = events.length;
|
||||
|
||||
delayedUpdate?.({
|
||||
content: [{ type: "text", text: "late" }],
|
||||
details: { status: "late" },
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
expect(events.filter((event) => event.type === "tool_execution_update")).toHaveLength(1);
|
||||
expect(events).toHaveLength(eventCountAfterPrompt);
|
||||
expect(unhandledRejections).toEqual([]);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandledRejection);
|
||||
}
|
||||
});
|
||||
|
||||
it("should ignore a settled parallel tool update while another tool is still running", async () => {
|
||||
const toolSchema = Type.Object({});
|
||||
const slowStarted = createDeferred();
|
||||
const settledToolEnded = createDeferred();
|
||||
const releaseSlow = createDeferred();
|
||||
let settledToolUpdate: AgentToolUpdateCallback<{ status: string }> | undefined;
|
||||
const events: AgentEvent[] = [];
|
||||
const settledTool: AgentTool<typeof toolSchema, { status: string }> = {
|
||||
name: "settled_tool",
|
||||
label: "Settled Tool",
|
||||
description: "Captures progress callbacks",
|
||||
parameters: toolSchema,
|
||||
async execute(_toolCallId, _params, _signal, onUpdate) {
|
||||
settledToolUpdate = onUpdate;
|
||||
return {
|
||||
content: [{ type: "text", text: "done" }],
|
||||
details: { status: "done" },
|
||||
terminate: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const slowTool: AgentTool<typeof toolSchema, { status: string }> = {
|
||||
name: "slow_tool",
|
||||
label: "Slow Tool",
|
||||
description: "Keeps the agent run active",
|
||||
parameters: toolSchema,
|
||||
async execute() {
|
||||
slowStarted.resolve();
|
||||
await releaseSlow.promise;
|
||||
return {
|
||||
content: [{ type: "text", text: "done" }],
|
||||
details: { status: "done" },
|
||||
terminate: true,
|
||||
};
|
||||
},
|
||||
};
|
||||
const agent = new Agent({
|
||||
initialState: { tools: [settledTool, slowTool] },
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({
|
||||
type: "done",
|
||||
reason: "toolUse",
|
||||
message: createAssistantToolUseMessage([
|
||||
{ type: "toolCall", id: "call-1", name: "settled_tool", arguments: {} },
|
||||
{ type: "toolCall", id: "call-2", name: "slow_tool", arguments: {} },
|
||||
]),
|
||||
});
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
agent.subscribe((event) => {
|
||||
events.push(event);
|
||||
if (event.type === "tool_execution_end" && event.toolCallId === "call-1") {
|
||||
settledToolEnded.resolve();
|
||||
}
|
||||
});
|
||||
|
||||
const promptPromise = agent.prompt("run tools");
|
||||
await Promise.all([slowStarted.promise, settledToolEnded.promise]);
|
||||
const eventCountBeforeLateUpdate = events.length;
|
||||
|
||||
settledToolUpdate?.({
|
||||
content: [{ type: "text", text: "late" }],
|
||||
details: { status: "late" },
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(events).toHaveLength(eventCountBeforeLateUpdate);
|
||||
|
||||
releaseSlow.resolve();
|
||||
await promptPromise;
|
||||
expect(events.filter((event) => event.type === "tool_execution_update")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("should update state with mutators", () => {
|
||||
const agent = new Agent({ streamFn: unusedStreamFunction });
|
||||
|
||||
// Test setSystemPrompt
|
||||
agent.state.systemPrompt = "Custom prompt";
|
||||
expect(agent.state.systemPrompt).toBe("Custom prompt");
|
||||
|
||||
// Test setModel
|
||||
const newModel = getModel("google", "gemini-2.5-flash");
|
||||
agent.state.model = newModel;
|
||||
expect(agent.state.model).toBe(newModel);
|
||||
|
||||
// Test setThinkingLevel
|
||||
agent.state.thinkingLevel = "high";
|
||||
expect(agent.state.thinkingLevel).toBe("high");
|
||||
|
||||
// Test setTools
|
||||
const tools = [{ name: "test", description: "test tool" } as any];
|
||||
agent.state.tools = tools;
|
||||
expect(agent.state.tools).toEqual(tools);
|
||||
expect(agent.state.tools).not.toBe(tools); // Should be a copy
|
||||
|
||||
// Test replaceMessages
|
||||
const messages = [{ role: "user" as const, content: "Hello", timestamp: Date.now() }];
|
||||
agent.state.messages = messages;
|
||||
expect(agent.state.messages).toEqual(messages);
|
||||
expect(agent.state.messages).not.toBe(messages); // Should be a copy
|
||||
|
||||
// Test appendMessage
|
||||
const newMessage = { role: "assistant" as const, content: [{ type: "text" as const, text: "Hi" }] };
|
||||
agent.state.messages.push(newMessage as any);
|
||||
expect(agent.state.messages).toHaveLength(2);
|
||||
expect(agent.state.messages[1]).toBe(newMessage);
|
||||
|
||||
// Test clearMessages
|
||||
agent.state.messages = [];
|
||||
expect(agent.state.messages).toEqual([]);
|
||||
});
|
||||
|
||||
it("should support steering message queue", async () => {
|
||||
const agent = new Agent({ streamFn: unusedStreamFunction });
|
||||
|
||||
const message = { role: "user" as const, content: "Steering message", timestamp: Date.now() };
|
||||
agent.steer(message);
|
||||
|
||||
// The message is queued but not yet in state.messages
|
||||
expect(agent.state.messages).not.toContainEqual(message);
|
||||
});
|
||||
|
||||
it("should support follow-up message queue", async () => {
|
||||
const agent = new Agent({ streamFn: unusedStreamFunction });
|
||||
|
||||
const message = { role: "user" as const, content: "Follow-up message", timestamp: Date.now() };
|
||||
agent.followUp(message);
|
||||
|
||||
// The message is queued but not yet in state.messages
|
||||
expect(agent.state.messages).not.toContainEqual(message);
|
||||
});
|
||||
|
||||
it("should handle abort controller", () => {
|
||||
const agent = new Agent({ streamFn: unusedStreamFunction });
|
||||
|
||||
// Should not throw even if nothing is running
|
||||
expect(() => agent.abort()).not.toThrow();
|
||||
});
|
||||
|
||||
it("should reject reset while processing without corrupting the transcript", async () => {
|
||||
const streamStarted = createDeferred();
|
||||
const releaseResponse = createDeferred();
|
||||
const agent = new Agent({
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(async () => {
|
||||
stream.push({ type: "start", partial: createAssistantMessage("") });
|
||||
streamStarted.resolve();
|
||||
await releaseResponse.promise;
|
||||
stream.push({ type: "done", reason: "stop", message: createAssistantMessage("Done") });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
const promptPromise = agent.prompt("Hello");
|
||||
await streamStarted.promise;
|
||||
|
||||
try {
|
||||
expect(agent.state.isStreaming).toBe(true);
|
||||
expect(agent.state.messages.map((message) => message.role)).toEqual(["user"]);
|
||||
expect(() => agent.reset()).toThrow("Agent is already processing. Wait for completion before resetting.");
|
||||
expect(agent.state.isStreaming).toBe(true);
|
||||
expect(agent.state.messages.map((message) => message.role)).toEqual(["user"]);
|
||||
} finally {
|
||||
releaseResponse.resolve();
|
||||
await promptPromise;
|
||||
}
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.map((message) => message.role)).toEqual(["user", "assistant"]);
|
||||
});
|
||||
|
||||
it("should throw when prompt() called while streaming", async () => {
|
||||
let abortSignal: AbortSignal | undefined;
|
||||
const agent = new Agent({
|
||||
// Use a stream function that responds to abort
|
||||
streamFn: (_model, _context, options) => {
|
||||
abortSignal = options?.signal;
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "start", partial: createAssistantMessage("") });
|
||||
// Check abort signal periodically
|
||||
const checkAbort = () => {
|
||||
if (abortSignal?.aborted) {
|
||||
stream.push({ type: "error", reason: "aborted", error: createAssistantMessage("Aborted") });
|
||||
} else {
|
||||
setTimeout(checkAbort, 5);
|
||||
}
|
||||
};
|
||||
checkAbort();
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
// Start first prompt (don't await, it will block until abort)
|
||||
const firstPrompt = agent.prompt("First message");
|
||||
|
||||
// Wait a tick for isStreaming to be set
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(agent.state.isStreaming).toBe(true);
|
||||
|
||||
// Second prompt should reject
|
||||
await expect(agent.prompt("Second message")).rejects.toThrow(
|
||||
"Agent is already processing a prompt. Use steer() or followUp() to queue messages, or wait for completion.",
|
||||
);
|
||||
|
||||
// Cleanup - abort to stop the stream
|
||||
agent.abort();
|
||||
await firstPrompt.catch(() => {}); // Ignore abort error
|
||||
});
|
||||
|
||||
it("should throw when continue() called while streaming", async () => {
|
||||
let abortSignal: AbortSignal | undefined;
|
||||
const agent = new Agent({
|
||||
streamFn: (_model, _context, options) => {
|
||||
abortSignal = options?.signal;
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "start", partial: createAssistantMessage("") });
|
||||
const checkAbort = () => {
|
||||
if (abortSignal?.aborted) {
|
||||
stream.push({ type: "error", reason: "aborted", error: createAssistantMessage("Aborted") });
|
||||
} else {
|
||||
setTimeout(checkAbort, 5);
|
||||
}
|
||||
};
|
||||
checkAbort();
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
// Start first prompt
|
||||
const firstPrompt = agent.prompt("First message");
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
expect(agent.state.isStreaming).toBe(true);
|
||||
|
||||
// continue() should reject
|
||||
await expect(agent.continue()).rejects.toThrow(
|
||||
"Agent is already processing. Wait for completion before continuing.",
|
||||
);
|
||||
|
||||
// Cleanup
|
||||
agent.abort();
|
||||
await firstPrompt.catch(() => {});
|
||||
});
|
||||
|
||||
it("continue() should process queued follow-up messages after an assistant turn", async () => {
|
||||
const agent = new Agent({
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "done", reason: "stop", message: createAssistantMessage("Processed") });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
agent.state.messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Initial" }],
|
||||
timestamp: Date.now() - 10,
|
||||
},
|
||||
createAssistantMessage("Initial response"),
|
||||
];
|
||||
|
||||
agent.followUp({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Queued follow-up" }],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
|
||||
await expect(agent.continue()).resolves.toBeUndefined();
|
||||
|
||||
const hasQueuedFollowUp = agent.state.messages.some((message) => {
|
||||
if (message.role !== "user") return false;
|
||||
if (typeof message.content === "string") return message.content === "Queued follow-up";
|
||||
return message.content.some((part) => part.type === "text" && part.text === "Queued follow-up");
|
||||
});
|
||||
|
||||
expect(hasQueuedFollowUp).toBe(true);
|
||||
expect(agent.state.messages[agent.state.messages.length - 1].role).toBe("assistant");
|
||||
});
|
||||
|
||||
it("continue() should keep one-at-a-time steering semantics from assistant tail", async () => {
|
||||
let responseCount = 0;
|
||||
const agent = new Agent({
|
||||
streamFn: () => {
|
||||
const stream = new MockAssistantStream();
|
||||
responseCount++;
|
||||
queueMicrotask(() => {
|
||||
stream.push({
|
||||
type: "done",
|
||||
reason: "stop",
|
||||
message: createAssistantMessage(`Processed ${responseCount}`),
|
||||
});
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
agent.state.messages = [
|
||||
{
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Initial" }],
|
||||
timestamp: Date.now() - 10,
|
||||
},
|
||||
createAssistantMessage("Initial response"),
|
||||
];
|
||||
|
||||
agent.steer({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Steering 1" }],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
agent.steer({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Steering 2" }],
|
||||
timestamp: Date.now() + 1,
|
||||
});
|
||||
|
||||
await expect(agent.continue()).resolves.toBeUndefined();
|
||||
|
||||
const recentMessages = agent.state.messages.slice(-4);
|
||||
expect(recentMessages.map((m) => m.role)).toEqual(["user", "assistant", "user", "assistant"]);
|
||||
expect(responseCount).toBe(2);
|
||||
});
|
||||
|
||||
it("keeps legacy prepareNextTurn signal callback behavior", async () => {
|
||||
const schema = Type.Object({});
|
||||
const tool: AgentTool<typeof schema> = {
|
||||
name: "noop",
|
||||
label: "Noop",
|
||||
description: "Noop tool",
|
||||
parameters: schema,
|
||||
execute: async () => ({ content: [{ type: "text", text: "ok" }], details: {} }),
|
||||
};
|
||||
let requestCount = 0;
|
||||
let sawAbortSignal = false;
|
||||
const agent = new Agent({
|
||||
initialState: { tools: [tool] },
|
||||
prepareNextTurn: async (signal) => {
|
||||
sawAbortSignal = signal instanceof AbortSignal;
|
||||
return undefined;
|
||||
},
|
||||
streamFn: () => {
|
||||
requestCount++;
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
if (requestCount === 1) {
|
||||
const message = createAssistantToolUseMessage([
|
||||
{ type: "toolCall", id: "tool-1", name: "noop", arguments: {} },
|
||||
]);
|
||||
stream.push({ type: "done", reason: "toolUse", message });
|
||||
return;
|
||||
}
|
||||
const message = createAssistantMessage("done");
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
await agent.prompt("start");
|
||||
|
||||
expect(requestCount).toBe(2);
|
||||
expect(sawAbortSignal).toBe(true);
|
||||
});
|
||||
|
||||
it("forwards shouldStopAfterTurn through AgentOptions", async () => {
|
||||
const schema = Type.Object({});
|
||||
const tool: AgentTool<typeof schema> = {
|
||||
name: "noop",
|
||||
label: "Noop",
|
||||
description: "Noop tool",
|
||||
parameters: schema,
|
||||
execute: async () => ({ content: [{ type: "text", text: "tool complete" }], details: {} }),
|
||||
};
|
||||
let requestCount = 0;
|
||||
let sawAbortSignal = false;
|
||||
let callbackContextRoles: string[] = [];
|
||||
const agent = new Agent({
|
||||
initialState: { tools: [tool] },
|
||||
shouldStopAfterTurn: (context, signal) => {
|
||||
sawAbortSignal = signal instanceof AbortSignal;
|
||||
callbackContextRoles = context.context.messages.map((message) => message.role);
|
||||
return true;
|
||||
},
|
||||
streamFn: () => {
|
||||
requestCount++;
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
if (requestCount === 1) {
|
||||
const message = createAssistantToolUseMessage([
|
||||
{ type: "toolCall", id: "tool-1", name: "noop", arguments: {} },
|
||||
]);
|
||||
stream.push({ type: "done", reason: "toolUse", message });
|
||||
return;
|
||||
}
|
||||
const message = createAssistantMessage("should not run");
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
await agent.prompt("start");
|
||||
|
||||
expect(requestCount).toBe(1);
|
||||
expect(sawAbortSignal).toBe(true);
|
||||
expect(callbackContextRoles).toEqual(["user", "assistant", "toolResult"]);
|
||||
});
|
||||
|
||||
it("forwards sessionId to streamFunction options", async () => {
|
||||
let receivedSessionId: string | undefined;
|
||||
const agent = new Agent({
|
||||
sessionId: "session-abc",
|
||||
streamFn: (_model, _context, options) => {
|
||||
receivedSessionId = options?.sessionId;
|
||||
const stream = new MockAssistantStream();
|
||||
queueMicrotask(() => {
|
||||
const message = createAssistantMessage("ok");
|
||||
stream.push({ type: "done", reason: "stop", message });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
});
|
||||
|
||||
await agent.prompt("hello");
|
||||
expect(receivedSessionId).toBe("session-abc");
|
||||
|
||||
// Test setter
|
||||
agent.sessionId = "session-def";
|
||||
expect(agent.sessionId).toBe("session-def");
|
||||
|
||||
await agent.prompt("hello again");
|
||||
expect(receivedSessionId).toBe("session-def");
|
||||
});
|
||||
});
|
||||
415
third_party/pi/packages/agent/test/e2e.test.ts
vendored
Normal file
415
third_party/pi/packages/agent/test/e2e.test.ts
vendored
Normal file
@@ -0,0 +1,415 @@
|
||||
import {
|
||||
type AssistantMessage,
|
||||
type FauxProviderRegistration,
|
||||
fauxAssistantMessage,
|
||||
fauxText,
|
||||
fauxThinking,
|
||||
fauxToolCall,
|
||||
type Model,
|
||||
registerFauxProvider,
|
||||
streamSimple,
|
||||
type ToolResultMessage,
|
||||
type UserMessage,
|
||||
} from "@earendil-works/pi-ai/compat";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { Agent, type AgentEvent } from "../src/index.ts";
|
||||
import { calculateTool } from "./utils/calculate.ts";
|
||||
|
||||
const registrations: FauxProviderRegistration[] = [];
|
||||
|
||||
function createFauxRegistration(options: Parameters<typeof registerFauxProvider>[0] = {}): FauxProviderRegistration {
|
||||
const registration = registerFauxProvider(options);
|
||||
registrations.push(registration);
|
||||
return registration;
|
||||
}
|
||||
|
||||
function getTextContent(message: AssistantMessage | ToolResultMessage): string {
|
||||
return message.content
|
||||
.filter((block) => block.type === "text")
|
||||
.map((block) => block.text)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (registrations.length > 0) {
|
||||
registrations.pop()?.unregister();
|
||||
}
|
||||
});
|
||||
|
||||
async function basicPrompt(model: Model<string>) {
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant. Keep your responses concise.",
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: [],
|
||||
},
|
||||
});
|
||||
|
||||
await agent.prompt("What is 2+2? Answer with just the number.");
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.length).toBe(2);
|
||||
expect(agent.state.messages[0].role).toBe("user");
|
||||
expect(agent.state.messages[1].role).toBe("assistant");
|
||||
|
||||
const assistantMessage = agent.state.messages[1];
|
||||
if (assistantMessage.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(getTextContent(assistantMessage)).toContain("4");
|
||||
}
|
||||
|
||||
async function toolExecution(model: Model<string>) {
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant. Always use the calculator tool for math.",
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: [calculateTool],
|
||||
},
|
||||
});
|
||||
|
||||
const pendingToolCallsDuringEvents: Array<{ type: AgentEvent["type"]; ids: string[] }> = [];
|
||||
agent.subscribe((event) => {
|
||||
if (event.type === "tool_execution_start" || event.type === "tool_execution_end") {
|
||||
pendingToolCallsDuringEvents.push({
|
||||
type: event.type,
|
||||
ids: [...agent.state.pendingToolCalls],
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
await agent.prompt("Calculate 123 * 456 using the calculator tool.");
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.length).toBeGreaterThanOrEqual(4);
|
||||
const toolResultMsg = agent.state.messages.find((message) => message.role === "toolResult");
|
||||
expect(toolResultMsg).toBeDefined();
|
||||
if (toolResultMsg?.role !== "toolResult") throw new Error("Expected tool result message");
|
||||
expect(getTextContent(toolResultMsg)).toContain("123 * 456 = 56088");
|
||||
|
||||
const finalMessage = agent.state.messages[agent.state.messages.length - 1];
|
||||
if (finalMessage.role !== "assistant") throw new Error("Expected final assistant message");
|
||||
expect(getTextContent(finalMessage)).toContain("56088");
|
||||
expect(agent.state.pendingToolCalls.size).toBe(0);
|
||||
expect(pendingToolCallsDuringEvents).toEqual([
|
||||
{ type: "tool_execution_start", ids: ["calc-1"] },
|
||||
{ type: "tool_execution_end", ids: [] },
|
||||
]);
|
||||
}
|
||||
|
||||
async function abortExecution(model: Model<string>) {
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: [],
|
||||
},
|
||||
});
|
||||
|
||||
const promptPromise = agent.prompt("Count slowly from 1 to 20.");
|
||||
setTimeout(() => {
|
||||
agent.abort();
|
||||
}, 30);
|
||||
|
||||
await promptPromise;
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.length).toBeGreaterThanOrEqual(2);
|
||||
|
||||
const lastMessage = agent.state.messages[agent.state.messages.length - 1];
|
||||
if (lastMessage.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(lastMessage.stopReason).toBe("aborted");
|
||||
expect(lastMessage.errorMessage).toBeDefined();
|
||||
expect(agent.state.errorMessage).toBe(lastMessage.errorMessage);
|
||||
}
|
||||
|
||||
async function stateUpdates(model: Model<string>) {
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: [],
|
||||
},
|
||||
});
|
||||
|
||||
const events: AgentEvent["type"][] = [];
|
||||
agent.subscribe((event) => {
|
||||
events.push(event.type);
|
||||
});
|
||||
|
||||
await agent.prompt("Count from 1 to 5.");
|
||||
|
||||
expect(events).toContain("agent_start");
|
||||
expect(events).toContain("turn_start");
|
||||
expect(events).toContain("message_start");
|
||||
expect(events).toContain("message_update");
|
||||
expect(events).toContain("message_end");
|
||||
expect(events).toContain("turn_end");
|
||||
expect(events).toContain("agent_end");
|
||||
expect(events.indexOf("agent_start")).toBeLessThan(events.indexOf("message_start"));
|
||||
expect(events.indexOf("message_start")).toBeLessThan(events.indexOf("message_end"));
|
||||
expect(events.indexOf("message_end")).toBeLessThan(events.lastIndexOf("agent_end"));
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.length).toBe(2);
|
||||
}
|
||||
|
||||
async function multiTurnConversation(model: Model<string>) {
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: [],
|
||||
},
|
||||
});
|
||||
|
||||
await agent.prompt("My name is Alice.");
|
||||
expect(agent.state.messages.length).toBe(2);
|
||||
|
||||
await agent.prompt("What is my name?");
|
||||
expect(agent.state.messages.length).toBe(4);
|
||||
|
||||
const lastMessage = agent.state.messages[3];
|
||||
if (lastMessage.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(getTextContent(lastMessage).toLowerCase()).toContain("alice");
|
||||
}
|
||||
|
||||
describe("Agent integration with faux provider", () => {
|
||||
it("handles a basic text prompt", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
faux.setResponses([fauxAssistantMessage("4")]);
|
||||
await basicPrompt(faux.getModel());
|
||||
});
|
||||
|
||||
it("executes tools and tracks pending tool calls", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
faux.setResponses([
|
||||
fauxAssistantMessage(
|
||||
[
|
||||
fauxText("Let me calculate that."),
|
||||
fauxToolCall("calculate", { expression: "123 * 456" }, { id: "calc-1" }),
|
||||
],
|
||||
{ stopReason: "toolUse" },
|
||||
),
|
||||
fauxAssistantMessage("The result is 56088."),
|
||||
]);
|
||||
await toolExecution(faux.getModel());
|
||||
});
|
||||
|
||||
it("handles abort during streaming", async () => {
|
||||
const faux = createFauxRegistration({
|
||||
tokensPerSecond: 20,
|
||||
tokenSize: { min: 2, max: 2 },
|
||||
});
|
||||
faux.setResponses([
|
||||
fauxAssistantMessage(
|
||||
"one two three four five six seven eight nine ten eleven twelve thirteen fourteen fifteen",
|
||||
),
|
||||
]);
|
||||
await abortExecution(faux.getModel());
|
||||
});
|
||||
|
||||
it("emits lifecycle updates while streaming", async () => {
|
||||
const faux = createFauxRegistration({ tokenSize: { min: 1, max: 1 } });
|
||||
faux.setResponses([fauxAssistantMessage("1 2 3 4 5")]);
|
||||
await stateUpdates(faux.getModel());
|
||||
});
|
||||
|
||||
it("maintains context across multiple turns", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
faux.setResponses([
|
||||
fauxAssistantMessage("Nice to meet you, Alice."),
|
||||
(context) => {
|
||||
const hasAlice = context.messages.some((message) => {
|
||||
if (message.role !== "user") return false;
|
||||
if (typeof message.content === "string") return message.content.includes("Alice");
|
||||
return message.content.some((block) => block.type === "text" && block.text.includes("Alice"));
|
||||
});
|
||||
return fauxAssistantMessage(hasAlice ? "Your name is Alice." : "I do not know your name.");
|
||||
},
|
||||
]);
|
||||
await multiTurnConversation(faux.getModel());
|
||||
});
|
||||
|
||||
it("preserves thinking content blocks", async () => {
|
||||
const faux = createFauxRegistration({ models: [{ id: "faux-reasoning", reasoning: true }] });
|
||||
faux.setResponses([fauxAssistantMessage([fauxThinking("step by step"), fauxText("4")])]);
|
||||
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant.",
|
||||
model: faux.getModel(),
|
||||
thinkingLevel: "low",
|
||||
tools: [],
|
||||
},
|
||||
});
|
||||
|
||||
await agent.prompt("What is 2+2?");
|
||||
|
||||
const assistantMessage = agent.state.messages[1];
|
||||
if (assistantMessage?.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(assistantMessage.content).toEqual([
|
||||
{ type: "thinking", thinking: "step by step" },
|
||||
{ type: "text", text: "4" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("Agent.continue() with faux provider", () => {
|
||||
describe("validation", () => {
|
||||
it("throws when no messages in context", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "Test",
|
||||
model: faux.getModel(),
|
||||
},
|
||||
});
|
||||
|
||||
await expect(agent.continue()).rejects.toThrow("No messages to continue from");
|
||||
});
|
||||
|
||||
it("throws when last message is assistant", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
const model = faux.getModel();
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "Test",
|
||||
model,
|
||||
},
|
||||
});
|
||||
|
||||
const assistantMessage: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "Hello" }],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
agent.state.messages = [assistantMessage];
|
||||
|
||||
await expect(agent.continue()).rejects.toThrow("Cannot continue from message role: assistant");
|
||||
});
|
||||
});
|
||||
|
||||
describe("continue from user message", () => {
|
||||
it("continues and gets a response when last message is user", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
faux.setResponses([fauxAssistantMessage("HELLO WORLD")]);
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt: "You are a helpful assistant. Follow instructions exactly.",
|
||||
model: faux.getModel(),
|
||||
thinkingLevel: "off",
|
||||
tools: [],
|
||||
},
|
||||
});
|
||||
|
||||
const userMessage: UserMessage = {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "Say exactly: HELLO WORLD" }],
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
agent.state.messages = [userMessage];
|
||||
|
||||
await agent.continue();
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.length).toBe(2);
|
||||
expect(agent.state.messages[0].role).toBe("user");
|
||||
expect(agent.state.messages[1].role).toBe("assistant");
|
||||
|
||||
const assistantMsg = agent.state.messages[1];
|
||||
if (assistantMsg.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(getTextContent(assistantMsg).toUpperCase()).toContain("HELLO WORLD");
|
||||
});
|
||||
});
|
||||
|
||||
describe("continue from tool result", () => {
|
||||
it("continues and processes tool results", async () => {
|
||||
const faux = createFauxRegistration();
|
||||
const model = faux.getModel();
|
||||
faux.setResponses([fauxAssistantMessage("The answer is 8.")]);
|
||||
const agent = new Agent({
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
systemPrompt:
|
||||
"You are a helpful assistant. After getting a calculation result, state the answer clearly.",
|
||||
model,
|
||||
thinkingLevel: "off",
|
||||
tools: [calculateTool],
|
||||
},
|
||||
});
|
||||
|
||||
const userMessage: UserMessage = {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "What is 5 + 3?" }],
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
const assistantMessage: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [
|
||||
{ type: "text", text: "Let me calculate that." },
|
||||
{ type: "toolCall", id: "calc-1", name: "calculate", arguments: { expression: "5 + 3" } },
|
||||
],
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "toolUse",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
const toolResult: ToolResultMessage = {
|
||||
role: "toolResult",
|
||||
toolCallId: "calc-1",
|
||||
toolName: "calculate",
|
||||
content: [{ type: "text", text: "5 + 3 = 8" }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
agent.state.messages = [userMessage, assistantMessage, toolResult];
|
||||
|
||||
await agent.continue();
|
||||
|
||||
expect(agent.state.isStreaming).toBe(false);
|
||||
expect(agent.state.messages.length).toBeGreaterThanOrEqual(4);
|
||||
|
||||
const lastMessage = agent.state.messages[agent.state.messages.length - 1];
|
||||
expect(lastMessage.role).toBe("assistant");
|
||||
if (lastMessage.role !== "assistant") throw new Error("Expected assistant message");
|
||||
expect(getTextContent(lastMessage)).toContain("8");
|
||||
});
|
||||
});
|
||||
});
|
||||
76
third_party/pi/packages/agent/test/harness/adaptive-publisher.test.ts
vendored
Normal file
76
third_party/pi/packages/agent/test/harness/adaptive-publisher.test.ts
vendored
Normal file
@@ -0,0 +1,76 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AdaptivePublisher } from "../../src/harness/utils/adaptive-publisher.ts";
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe("AdaptivePublisher", () => {
|
||||
it("bounds event count and spaces large publications by encoded size", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
let value = "a";
|
||||
const updates: string[] = [];
|
||||
const publisher = new AdaptivePublisher({
|
||||
snapshot: () => value,
|
||||
update: (_previous, current) => current,
|
||||
measure: (update) => update.length,
|
||||
publish: (update) => updates.push(update),
|
||||
onError: (error) => {
|
||||
throw error;
|
||||
},
|
||||
minIntervalMs: 100,
|
||||
targetBytesPerSecond: 100,
|
||||
});
|
||||
|
||||
publisher.markDirty();
|
||||
value = "x".repeat(100);
|
||||
publisher.markDirty();
|
||||
vi.advanceTimersByTime(100);
|
||||
expect(updates).toEqual(["a", "x".repeat(100)]);
|
||||
|
||||
value = "held";
|
||||
publisher.markDirty();
|
||||
vi.advanceTimersByTime(999);
|
||||
expect(updates).toHaveLength(2);
|
||||
vi.advanceTimersByTime(1);
|
||||
expect(updates).toEqual(["a", "x".repeat(100), "held"]);
|
||||
});
|
||||
|
||||
it("commits its baseline before a consumer throws", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
let value = "a";
|
||||
const updates: Array<{ previous: string | undefined; current: string }> = [];
|
||||
let throwAfterApply = false;
|
||||
const publisher = new AdaptivePublisher({
|
||||
snapshot: () => value,
|
||||
update: (previous, current) => ({ previous, current }),
|
||||
measure: () => 1,
|
||||
publish: (update) => {
|
||||
updates.push(update);
|
||||
if (throwAfterApply) throw new Error("consumer failed after apply");
|
||||
},
|
||||
onError: () => {},
|
||||
minIntervalMs: 100,
|
||||
targetBytesPerSecond: 100,
|
||||
});
|
||||
|
||||
publisher.markDirty();
|
||||
vi.advanceTimersByTime(100);
|
||||
value = "ab";
|
||||
throwAfterApply = true;
|
||||
expect(() => publisher.markDirty()).toThrow("consumer failed after apply");
|
||||
publisher.flush(true);
|
||||
expect(updates).toEqual([
|
||||
{ previous: undefined, current: "a" },
|
||||
{ previous: "a", current: "ab" },
|
||||
]);
|
||||
|
||||
throwAfterApply = false;
|
||||
vi.advanceTimersByTime(100);
|
||||
value = "abc";
|
||||
publisher.markDirty();
|
||||
expect(updates.at(-1)).toEqual({ previous: "ab", current: "abc" });
|
||||
});
|
||||
});
|
||||
71
third_party/pi/packages/agent/test/harness/branch-summarization.test.ts
vendored
Normal file
71
third_party/pi/packages/agent/test/harness/branch-summarization.test.ts
vendored
Normal file
@@ -0,0 +1,71 @@
|
||||
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { collectEntriesForBranchSummary } from "../../src/harness/compaction/branch-summarization.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import type { Branch, Entry, MessageEntry, Session } from "../../src/harness/session/index.ts";
|
||||
|
||||
function message(text: string): AgentMessage {
|
||||
return { role: "user", content: [{ type: "text", text }], timestamp: 1 };
|
||||
}
|
||||
|
||||
function messageEntry(id: string, parentId: string | null, text: string, seq: number): MessageEntry {
|
||||
return { type: "message", id, parentId, message: message(text), seq, timestamp: seq };
|
||||
}
|
||||
|
||||
function branchReader(entries: Entry[]): {
|
||||
branch: Pick<Branch, "findEntries">;
|
||||
session: Pick<Session, "getEntry">;
|
||||
} {
|
||||
const byId = new Map(entries.map((entry) => [entry.id, entry]));
|
||||
return {
|
||||
session: {
|
||||
async getEntry(id) {
|
||||
return byId.get(id);
|
||||
},
|
||||
},
|
||||
branch: {
|
||||
async findEntries(query = {}) {
|
||||
const path: Entry[] = [];
|
||||
let currentId = query.start ?? null;
|
||||
while (currentId !== null) {
|
||||
const entry = byId.get(currentId);
|
||||
if (!entry) throw new Error(`Unknown entry ${currentId}`);
|
||||
path.push(entry);
|
||||
currentId = entry.parentId;
|
||||
}
|
||||
return path;
|
||||
},
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
describe("v4 branch summarization", () => {
|
||||
it("collects the abandoned side of a branch in chronological order", async () => {
|
||||
const root = messageEntry("root", null, "root", 1);
|
||||
const common = messageEntry("common", root.id, "common", 2);
|
||||
const abandoned1 = messageEntry("abandoned-1", common.id, "abandoned 1", 3);
|
||||
const abandoned2 = messageEntry("abandoned-2", abandoned1.id, "abandoned 2", 4);
|
||||
const target = messageEntry("target", common.id, "target", 5);
|
||||
const { branch, session } = branchReader([root, common, abandoned1, abandoned2, target]);
|
||||
|
||||
const result = await collectEntriesForBranchSummary(
|
||||
branch,
|
||||
session,
|
||||
abandoned2.id,
|
||||
target.id,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result.commonAncestorId).toBe(common.id);
|
||||
expect(result.entries.map((entry) => entry.id)).toEqual([abandoned1.id, abandoned2.id]);
|
||||
expect(result.entries.some((entry) => entry.id === root.id)).toBe(false);
|
||||
});
|
||||
|
||||
it("returns no entries when there was no previous leaf", async () => {
|
||||
const target = messageEntry("target", null, "target", 1);
|
||||
const { branch, session } = branchReader([target]);
|
||||
expect(await collectEntriesForBranchSummary(branch, session, null, target.id, BACKGROUND_CONTEXT)).toEqual({
|
||||
entries: [],
|
||||
commonAncestorId: null,
|
||||
});
|
||||
});
|
||||
});
|
||||
110
third_party/pi/packages/agent/test/harness/branch.test.ts
vendored
Normal file
110
third_party/pi/packages/agent/test/harness/branch.test.ts
vendored
Normal file
@@ -0,0 +1,110 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { insertEntry } from "../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import { SessionPendingAssistantMessageError, StorageBackedSession } from "../../src/harness/session/session.ts";
|
||||
import { branchTip, setValue } from "../../src/harness/session/values.ts";
|
||||
|
||||
function createSession(): StorageBackedSession {
|
||||
return new StorageBackedSession(
|
||||
{ id: "session", createdAt: 1, storageVersion: 1 },
|
||||
new MemoryStorage({ now: () => 10 }),
|
||||
{ idGenerator: { next: () => `entry-${nextId++}` } },
|
||||
);
|
||||
}
|
||||
|
||||
let nextId = 1;
|
||||
|
||||
describe("Branch", () => {
|
||||
it("is absent until explicitly created and Session has no implicit Branch surface", async () => {
|
||||
const session = createSession();
|
||||
expect(await session.branch("main", BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect("getTipId" in session).toBe(false);
|
||||
expect("appendMessage" in session).toBe(false);
|
||||
expect("findEntriesOnBranch" in session).toBe(false);
|
||||
|
||||
const branch = await session.createBranch("main", null, BACKGROUND_CONTEXT);
|
||||
expect(branch.name).toBe("main");
|
||||
expect(await branch.getTipId(BACKGROUND_CONTEXT)).toBeNull();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("directly appends immutable entries and advances only its own tip", async () => {
|
||||
nextId = 1;
|
||||
const session = createSession();
|
||||
const main = await session.createBranch("main", null, BACKGROUND_CONTEXT);
|
||||
const review = await session.createBranch("review", null, BACKGROUND_CONTEXT);
|
||||
const message = { role: "user" as const, content: "hello", timestamp: 1 };
|
||||
|
||||
const messageId = await main.appendMessage(message, BACKGROUND_CONTEXT);
|
||||
const customId = await main.appendCustomEntry("note", { ok: true }, BACKGROUND_CONTEXT);
|
||||
const reviewId = await review.appendCustomEntry("review", undefined, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await main.getTipId(BACKGROUND_CONTEXT)).toBe(customId);
|
||||
expect(await review.getTipId(BACKGROUND_CONTEXT)).toBe(reviewId);
|
||||
expect((await main.findEntries({ order: "oldestFirst" }, BACKGROUND_CONTEXT)).map(({ id }) => id)).toEqual([
|
||||
messageId,
|
||||
customId,
|
||||
]);
|
||||
expect(await main.findEntry({ type: "message" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
id: messageId,
|
||||
type: "message",
|
||||
message,
|
||||
});
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("supports explicit starts without moving the Branch tip", async () => {
|
||||
const session = createSession();
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
insertEntry({ id: "root", parentId: null, type: "custom", customType: "root" }),
|
||||
insertEntry({ id: "left", parentId: "root", type: "custom", customType: "left" }),
|
||||
insertEntry({ id: "right", parentId: "root", type: "custom", customType: "right" }),
|
||||
setValue(branchTip("main"), "left"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const branch = await session.branch("main", BACKGROUND_CONTEXT);
|
||||
if (branch === undefined) throw new Error("Expected main Branch");
|
||||
|
||||
expect(
|
||||
(await branch.findEntries({ start: "right", order: "oldestFirst" }, BACKGROUND_CONTEXT)).map(({ id }) => id),
|
||||
).toEqual(["root", "right"]);
|
||||
expect(await branch.getTipId(BACKGROUND_CONTEXT)).toBe("left");
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects pending assistant messages before committing", async () => {
|
||||
const session = createSession();
|
||||
const branch = await session.createBranch("main", null, BACKGROUND_CONTEXT);
|
||||
await expect(
|
||||
branch.appendMessage(
|
||||
{
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "test",
|
||||
provider: "test",
|
||||
model: "test",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "pending",
|
||||
timestamp: 1,
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toBeInstanceOf(SessionPendingAssistantMessageError);
|
||||
expect(await branch.getTipId(BACKGROUND_CONTEXT)).toBeNull();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
827
third_party/pi/packages/agent/test/harness/compaction.test.ts
vendored
Normal file
827
third_party/pi/packages/agent/test/harness/compaction.test.ts
vendored
Normal file
@@ -0,0 +1,827 @@
|
||||
import {
|
||||
type Api,
|
||||
type AssistantMessage,
|
||||
createModels,
|
||||
type FauxProviderHandle,
|
||||
fauxAssistantMessage,
|
||||
fauxProvider,
|
||||
type Message,
|
||||
type Model,
|
||||
type Models,
|
||||
type Usage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { beforeEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
type CompactionPreparation,
|
||||
type CompactionSettings,
|
||||
calculateContextTokens,
|
||||
compact,
|
||||
DEFAULT_COMPACTION_SETTINGS,
|
||||
estimateContextTokens,
|
||||
estimateTokens,
|
||||
findCutPoint,
|
||||
findTurnStartIndex,
|
||||
generateSummary,
|
||||
generateSummaryWithUsage,
|
||||
getLastAssistantUsage,
|
||||
prepareCompaction,
|
||||
serializeConversation,
|
||||
shouldCompact,
|
||||
} from "../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { buildSessionContext } from "../../src/harness/session/context.ts";
|
||||
import type {
|
||||
BranchSummaryEntry,
|
||||
CompactionEntry,
|
||||
CustomEntry,
|
||||
Entry,
|
||||
MessageEntry,
|
||||
} from "../../src/harness/session/types.ts";
|
||||
import { getOrThrow } from "../../src/harness/types.ts";
|
||||
import type { AgentMessage } from "../../src/types.ts";
|
||||
|
||||
let nextId = 0;
|
||||
function createId(): string {
|
||||
return `entry-${nextId++}`;
|
||||
}
|
||||
|
||||
function createMockUsage(input: number, output: number, cacheRead = 0, cacheWrite = 0): Usage {
|
||||
return {
|
||||
input,
|
||||
output,
|
||||
cacheRead,
|
||||
cacheWrite,
|
||||
totalTokens: input + output + cacheRead + cacheWrite,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function createUserMessage(text: string): AgentMessage {
|
||||
return {
|
||||
role: "user",
|
||||
content: [{ type: "text", text }],
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function createAssistantMessage(text: string, usage = createMockUsage(100, 50)): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text }],
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-5",
|
||||
usage,
|
||||
stopReason: "stop",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
function createMessageEntry(message: AgentMessage, parentId: string | null = null): MessageEntry {
|
||||
return {
|
||||
type: "message",
|
||||
id: createId(),
|
||||
parentId,
|
||||
seq: nextId,
|
||||
timestamp: Date.now(),
|
||||
message,
|
||||
};
|
||||
}
|
||||
|
||||
function createCompactionEntry(
|
||||
summary: string,
|
||||
parentId: string | null = null,
|
||||
retainedTail?: AgentMessage[],
|
||||
): CompactionEntry {
|
||||
return {
|
||||
type: "compaction",
|
||||
id: createId(),
|
||||
parentId,
|
||||
seq: nextId,
|
||||
timestamp: Date.now(),
|
||||
summary,
|
||||
tokensBefore: 1234,
|
||||
retainedTail: retainedTail ?? [],
|
||||
fromHook: false,
|
||||
};
|
||||
}
|
||||
|
||||
function createCustomEntry(customType: string, parentId: string | null = null): CustomEntry {
|
||||
return {
|
||||
type: "custom",
|
||||
customType,
|
||||
id: createId(),
|
||||
parentId,
|
||||
seq: nextId,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
}
|
||||
|
||||
/** Shared collection; each faux provider gets a unique id so coexisting fakes route correctly. */
|
||||
const models = createModels();
|
||||
let fauxCount = 0;
|
||||
|
||||
function createFauxModel(reasoning: boolean, maxTokens = 8192): { faux: FauxProviderHandle; model: Model<Api> } {
|
||||
const faux = fauxProvider({
|
||||
provider: `faux-${++fauxCount}`,
|
||||
models: [
|
||||
{
|
||||
id: reasoning ? "reasoning-model" : "non-reasoning-model",
|
||||
reasoning,
|
||||
contextWindow: 200000,
|
||||
maxTokens,
|
||||
},
|
||||
],
|
||||
});
|
||||
models.setProvider(faux.provider);
|
||||
return { faux, model: faux.getModel() };
|
||||
}
|
||||
|
||||
function createModelsWithSimpleResponses(responses: AssistantMessage[]): Models {
|
||||
const remaining = [...responses];
|
||||
const stub = Object.create(models) as Models;
|
||||
stub.completeSimple = async () => {
|
||||
const response = remaining.shift();
|
||||
if (!response) throw new Error("No faux completeSimple response queued");
|
||||
return response;
|
||||
};
|
||||
return stub;
|
||||
}
|
||||
|
||||
describe("harness compaction", () => {
|
||||
beforeEach(() => {
|
||||
nextId = 0;
|
||||
});
|
||||
|
||||
it("calculates total context tokens from usage", () => {
|
||||
expect(calculateContextTokens(createMockUsage(1000, 500, 200, 100))).toBe(1800);
|
||||
expect(calculateContextTokens(createMockUsage(0, 0, 0, 0))).toBe(0);
|
||||
});
|
||||
|
||||
it("checks compaction threshold", () => {
|
||||
const settings: CompactionSettings = {
|
||||
enabled: true,
|
||||
reserveTokens: 10000,
|
||||
keepRecentTokens: 20000,
|
||||
};
|
||||
expect(shouldCompact(95000, 100000, settings)).toBe(true);
|
||||
expect(shouldCompact(89000, 100000, settings)).toBe(false);
|
||||
expect(shouldCompact(95000, 100000, { ...settings, enabled: false })).toBe(false);
|
||||
});
|
||||
|
||||
it("finds a cut point based on token differences", () => {
|
||||
const entries: Entry[] = [];
|
||||
let parentId: string | null = null;
|
||||
for (let i = 0; i < 10; i++) {
|
||||
const user = createMessageEntry(createUserMessage(`User ${i}`), parentId);
|
||||
entries.push(user);
|
||||
const assistant = createMessageEntry(
|
||||
createAssistantMessage(`Assistant ${i}`, createMockUsage(0, 100, (i + 1) * 1000, 0)),
|
||||
user.id,
|
||||
);
|
||||
entries.push(assistant);
|
||||
parentId = assistant.id;
|
||||
}
|
||||
|
||||
const result = findCutPoint(entries, 0, entries.length, 2500);
|
||||
expect(entries[result.firstKeptEntryIndex]?.type).toBe("message");
|
||||
});
|
||||
|
||||
it("covers cut-point and turn-start edge cases", () => {
|
||||
const firstCustom = createCustomEntry("first");
|
||||
const secondCustom = createCustomEntry("second", firstCustom.id);
|
||||
expect(findCutPoint([firstCustom, secondCustom], 0, 2, 1)).toEqual({
|
||||
firstKeptEntryIndex: 0,
|
||||
turnStartIndex: -1,
|
||||
isSplitTurn: false,
|
||||
});
|
||||
|
||||
const branchSummary: BranchSummaryEntry = {
|
||||
type: "branch_summary",
|
||||
id: createId(),
|
||||
parentId: secondCustom.id,
|
||||
seq: nextId,
|
||||
timestamp: Date.now(),
|
||||
fromId: "branch",
|
||||
summary: "branch summary",
|
||||
fromHook: false,
|
||||
};
|
||||
expect(findTurnStartIndex([firstCustom, branchSummary], 1, 0)).toBe(1);
|
||||
expect(findTurnStartIndex([firstCustom, secondCustom], 1, 0)).toBe(-1);
|
||||
|
||||
const result = findCutPoint([firstCustom, branchSummary], 0, 2, 1);
|
||||
expect(result.firstKeptEntryIndex).toBe(0);
|
||||
|
||||
const toolResult = createMessageEntry({
|
||||
role: "toolResult",
|
||||
toolCallId: "call-1",
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: "tool output" }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
expect(findCutPoint([toolResult], 0, 1, 1)).toEqual({
|
||||
firstKeptEntryIndex: 0,
|
||||
turnStartIndex: -1,
|
||||
isSplitTurn: false,
|
||||
});
|
||||
|
||||
const user = createMessageEntry(createUserMessage("user"));
|
||||
const compaction = createCompactionEntry("summary", user.id);
|
||||
const assistant = createMessageEntry(createAssistantMessage("assistant"), compaction.id);
|
||||
expect(findCutPoint([user, compaction, assistant], 0, 3, 1).firstKeptEntryIndex).toBe(2);
|
||||
});
|
||||
|
||||
it("estimates tokens and context usage across supported message roles", () => {
|
||||
const usage = createMockUsage(10, 5, 3, 2);
|
||||
const assistant = createAssistantMessage("assistant", usage);
|
||||
const assistantWithThinkingAndTool: AssistantMessage = {
|
||||
...assistant,
|
||||
content: [
|
||||
{ type: "thinking", thinking: "thinking" },
|
||||
{ type: "toolCall", id: "call-1", name: "read", arguments: { path: "file.ts" } },
|
||||
],
|
||||
};
|
||||
const customString: AgentMessage = {
|
||||
role: "custom",
|
||||
customType: "note",
|
||||
content: "custom text",
|
||||
display: true,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
const toolResultWithImage: AgentMessage = {
|
||||
role: "toolResult",
|
||||
toolCallId: "call-1",
|
||||
toolName: "read",
|
||||
content: [
|
||||
{ type: "text", text: "tool text" },
|
||||
{ type: "image", mimeType: "image/png", data: "abc" },
|
||||
],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
const bashExecution: AgentMessage = {
|
||||
role: "bashExecution",
|
||||
command: "npm run check",
|
||||
output: "ok",
|
||||
exitCode: 0,
|
||||
cancelled: false,
|
||||
truncated: false,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
const branchSummaryMessage: AgentMessage = {
|
||||
role: "branchSummary",
|
||||
summary: "branch",
|
||||
fromId: "x",
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
const compactionSummaryMessage: AgentMessage = {
|
||||
role: "compactionSummary",
|
||||
summary: "compact",
|
||||
tokensBefore: 123,
|
||||
timestamp: Date.now(),
|
||||
};
|
||||
|
||||
expect(estimateTokens({ role: "user", content: "plain user", timestamp: Date.now() })).toBeGreaterThan(0);
|
||||
expect(estimateTokens(assistantWithThinkingAndTool)).toBeGreaterThan(0);
|
||||
expect(estimateTokens(customString)).toBeGreaterThan(0);
|
||||
expect(estimateTokens(toolResultWithImage)).toBeGreaterThan(1000);
|
||||
expect(estimateTokens(bashExecution)).toBeGreaterThan(0);
|
||||
expect(estimateTokens(branchSummaryMessage)).toBeGreaterThan(0);
|
||||
expect(estimateTokens(compactionSummaryMessage)).toBeGreaterThan(0);
|
||||
expect(estimateTokens({ role: "unknown", timestamp: Date.now() } as unknown as AgentMessage)).toBe(0);
|
||||
expect(
|
||||
getLastAssistantUsage([createMessageEntry(createUserMessage("user")), createMessageEntry(assistant)]),
|
||||
).toBe(usage);
|
||||
expect(
|
||||
getLastAssistantUsage([
|
||||
createMessageEntry({ ...assistant, stopReason: "aborted" }),
|
||||
createMessageEntry({ ...assistant, stopReason: "error" }),
|
||||
]),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
getLastAssistantUsage([
|
||||
createMessageEntry(createUserMessage("user")),
|
||||
createMessageEntry(assistant),
|
||||
createMessageEntry(createAssistantMessage("partial", createMockUsage(0, 0))),
|
||||
]),
|
||||
).toBe(usage);
|
||||
expect(estimateContextTokens([createUserMessage("no usage")]).lastUsageIndex).toBeNull();
|
||||
expect(estimateContextTokens([assistant, createUserMessage("tail")])).toMatchObject({
|
||||
usageTokens: 20,
|
||||
lastUsageIndex: 0,
|
||||
});
|
||||
const estimate = estimateContextTokens([
|
||||
createUserMessage("Hello"),
|
||||
assistant,
|
||||
createUserMessage("continue"),
|
||||
createAssistantMessage("Partial thinking", createMockUsage(0, 0)),
|
||||
]);
|
||||
expect(estimate.usageTokens).toBe(20);
|
||||
expect(estimate.lastUsageIndex).toBe(1);
|
||||
expect(estimate.trailingTokens).toBeGreaterThan(0);
|
||||
expect(estimate.tokens).toBe(20 + estimate.trailingTokens);
|
||||
});
|
||||
|
||||
it("builds session context with a compaction entry", async () => {
|
||||
const u1 = createMessageEntry(createUserMessage("1"));
|
||||
const a1 = createMessageEntry(createAssistantMessage("a"), u1.id);
|
||||
const u2 = createMessageEntry(createUserMessage("2"), a1.id);
|
||||
const a2 = createMessageEntry(createAssistantMessage("b"), u2.id);
|
||||
const compaction = createCompactionEntry("Summary of 1,a,2,b", a2.id, [
|
||||
createUserMessage("2"),
|
||||
createAssistantMessage("b"),
|
||||
]);
|
||||
const u3 = createMessageEntry(createUserMessage("3"), compaction.id);
|
||||
const a3 = createMessageEntry(createAssistantMessage("c"), u3.id);
|
||||
const loaded = await buildSessionContext([u1, a1, u2, a2, compaction, u3, a3], undefined, BACKGROUND_CONTEXT);
|
||||
expect(loaded).toHaveLength(5);
|
||||
expect(loaded[0]?.role).toBe("compactionSummary");
|
||||
expect(loaded.map((message) => message.role)).toEqual([
|
||||
"compactionSummary",
|
||||
"user",
|
||||
"assistant",
|
||||
"user",
|
||||
"assistant",
|
||||
]);
|
||||
});
|
||||
|
||||
it("prepares compaction using the latest compaction summary as previousSummary", async () => {
|
||||
const u1 = createMessageEntry(createUserMessage("user msg 1"));
|
||||
const a1 = createMessageEntry(createAssistantMessage("assistant msg 1"), u1.id);
|
||||
const u2 = createMessageEntry(createUserMessage("user msg 2"), a1.id);
|
||||
const a2 = createMessageEntry(createAssistantMessage("assistant msg 2", createMockUsage(5000, 1000)), u2.id);
|
||||
const compaction1 = createCompactionEntry("First summary", a2.id);
|
||||
const u3 = createMessageEntry(createUserMessage("user msg 3"), compaction1.id);
|
||||
const a3 = createMessageEntry(createAssistantMessage("assistant msg 3", createMockUsage(8000, 2000)), u3.id);
|
||||
const pathEntries = [u1, a1, u2, a2, compaction1, u3, a3];
|
||||
const preparation = getOrThrow(prepareCompaction(pathEntries, DEFAULT_COMPACTION_SETTINGS));
|
||||
expect(preparation).toBeDefined();
|
||||
expect(preparation?.previousSummary).toBe("First summary");
|
||||
expect(preparation?.retainedTail.length).toBeGreaterThan(0);
|
||||
expect(preparation?.tokensBefore).toBe(
|
||||
estimateContextTokens(await buildSessionContext(pathEntries, undefined, BACKGROUND_CONTEXT)).tokens,
|
||||
);
|
||||
});
|
||||
|
||||
it("carries a previous compaction's retained tail into the next preparation", () => {
|
||||
const retainedUser = createUserMessage("retained user");
|
||||
const retainedAssistant = createAssistantMessage("retained assistant");
|
||||
const compaction = createCompactionEntry("previous summary", null, [retainedUser, retainedAssistant]);
|
||||
const user = createMessageEntry(createUserMessage("new user"), compaction.id);
|
||||
const assistant = createMessageEntry(createAssistantMessage("new assistant"), user.id);
|
||||
|
||||
const preparation = getOrThrow(
|
||||
prepareCompaction([compaction, user, assistant], {
|
||||
enabled: true,
|
||||
reserveTokens: 100,
|
||||
keepRecentTokens: 1,
|
||||
}),
|
||||
);
|
||||
expect(preparation?.previousSummary).toBe("previous summary");
|
||||
expect([
|
||||
...(preparation?.messagesToSummarize ?? []),
|
||||
...(preparation?.turnPrefixMessages ?? []),
|
||||
...(preparation?.retainedTail ?? []),
|
||||
]).toEqual([retainedUser, retainedAssistant, user.message, assistant.message]);
|
||||
});
|
||||
|
||||
it("prepares split-turn compaction with prior file-operation details", () => {
|
||||
const u1 = createMessageEntry(createUserMessage("user msg 1"));
|
||||
const assistantMessage: AssistantMessage = {
|
||||
...createAssistantMessage("assistant msg 1"),
|
||||
content: [{ type: "toolCall", id: "tool-1", name: "write", arguments: { path: "written.ts" } }],
|
||||
};
|
||||
const a1 = createMessageEntry(assistantMessage, u1.id);
|
||||
const compaction1: CompactionEntry = {
|
||||
...createCompactionEntry("First summary", a1.id),
|
||||
details: { readFiles: ["old-read.ts"], modifiedFiles: ["old-edit.ts", "written.ts"] },
|
||||
};
|
||||
const u2 = createMessageEntry(createUserMessage("large turn"), compaction1.id);
|
||||
const a2 = createMessageEntry(createAssistantMessage("large assistant message"), u2.id);
|
||||
const preparation = getOrThrow(
|
||||
prepareCompaction([u1, a1, compaction1, u2, a2], {
|
||||
enabled: true,
|
||||
reserveTokens: 100,
|
||||
keepRecentTokens: 1,
|
||||
}),
|
||||
);
|
||||
|
||||
expect(preparation).toMatchObject({ previousSummary: "First summary", isSplitTurn: true });
|
||||
expect(preparation?.turnPrefixMessages.map((message) => message.role)).toEqual(["user"]);
|
||||
expect([...preparation!.fileOps.read]).toContain("old-read.ts");
|
||||
expect([...preparation!.fileOps.edited]).toContain("old-edit.ts");
|
||||
expect([...preparation!.fileOps.edited]).toContain("written.ts");
|
||||
});
|
||||
|
||||
it("does not prepare compaction when there is nothing valid to compact", () => {
|
||||
const compaction = createCompactionEntry("already compacted");
|
||||
expect(getOrThrow(prepareCompaction([compaction], DEFAULT_COMPACTION_SETTINGS))).toBeUndefined();
|
||||
expect(getOrThrow(prepareCompaction([], DEFAULT_COMPACTION_SETTINGS))).toBeUndefined();
|
||||
});
|
||||
|
||||
it("serializes conversation with truncated tool results", () => {
|
||||
const longContent = "x".repeat(5000);
|
||||
const messages = convertMessages([
|
||||
{
|
||||
role: "toolResult",
|
||||
toolCallId: "tc1",
|
||||
toolName: "read",
|
||||
content: [{ type: "text", text: longContent }],
|
||||
isError: false,
|
||||
timestamp: Date.now(),
|
||||
},
|
||||
]);
|
||||
const result = serializeConversation(messages);
|
||||
expect(result).toContain("[Tool result]:");
|
||||
expect(result).toContain("[... 3000 more characters truncated]");
|
||||
});
|
||||
|
||||
it("passes reasoning through generateSummary only for reasoning models with thinking enabled", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const seenOptions: Array<Record<string, unknown> | undefined> = [];
|
||||
const { faux: fauxReasoning, model: reasoningModel } = createFauxModel(true);
|
||||
fauxReasoning.setResponses([
|
||||
(_context, options) => {
|
||||
seenOptions.push(options as Record<string, unknown> | undefined);
|
||||
return fauxAssistantMessage("## Goal\nTest summary");
|
||||
},
|
||||
]);
|
||||
getOrThrow(
|
||||
await generateSummary(
|
||||
messages,
|
||||
models,
|
||||
reasoningModel,
|
||||
2000,
|
||||
undefined,
|
||||
undefined,
|
||||
"medium",
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(seenOptions[0]).toMatchObject({ reasoning: "medium" });
|
||||
|
||||
const { faux: fauxOff, model: offModel } = createFauxModel(true);
|
||||
fauxOff.setResponses([
|
||||
(_context, options) => {
|
||||
seenOptions.push(options as Record<string, unknown> | undefined);
|
||||
return fauxAssistantMessage("## Goal\nTest summary");
|
||||
},
|
||||
]);
|
||||
getOrThrow(
|
||||
await generateSummary(
|
||||
messages,
|
||||
models,
|
||||
offModel,
|
||||
2000,
|
||||
undefined,
|
||||
undefined,
|
||||
"off",
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(seenOptions[1]).not.toHaveProperty("reasoning");
|
||||
|
||||
const { faux: fauxNonReasoning, model: nonReasoningModel } = createFauxModel(false);
|
||||
fauxNonReasoning.setResponses([
|
||||
(_context, options) => {
|
||||
seenOptions.push(options as Record<string, unknown> | undefined);
|
||||
return fauxAssistantMessage("## Goal\nTest summary");
|
||||
},
|
||||
]);
|
||||
getOrThrow(
|
||||
await generateSummary(
|
||||
messages,
|
||||
models,
|
||||
nonReasoningModel,
|
||||
2000,
|
||||
undefined,
|
||||
undefined,
|
||||
"medium",
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(seenOptions[2]).not.toHaveProperty("reasoning");
|
||||
});
|
||||
|
||||
it("includes previous summaries and custom instructions in generateSummary prompts", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
let promptText = "";
|
||||
const { faux, model } = createFauxModel(false);
|
||||
faux.setResponses([
|
||||
(context) => {
|
||||
const message = context.messages[0];
|
||||
const content = message?.role === "user" ? message.content : [];
|
||||
promptText = Array.isArray(content) && content[0]?.type === "text" ? content[0].text : "";
|
||||
return fauxAssistantMessage("## Goal\nTest summary");
|
||||
},
|
||||
]);
|
||||
|
||||
const summary = getOrThrow(
|
||||
await generateSummaryWithUsage(
|
||||
messages,
|
||||
models,
|
||||
model,
|
||||
2000,
|
||||
"focus",
|
||||
"old summary",
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
|
||||
expect(summary.text).toContain("Test summary");
|
||||
expect(summary.usage.input).toBeGreaterThan(0);
|
||||
expect(summary.usage.output).toBeGreaterThan(0);
|
||||
expect(summary.usage.totalTokens).toBe(
|
||||
summary.usage.input + summary.usage.output + summary.usage.cacheRead + summary.usage.cacheWrite,
|
||||
);
|
||||
expect(promptText).toContain("<previous-summary>\nold summary\n</previous-summary>");
|
||||
expect(promptText).toContain("Additional focus: focus");
|
||||
});
|
||||
|
||||
it("preserves the string result from generateSummary", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const { faux, model } = createFauxModel(false);
|
||||
faux.setResponses([fauxAssistantMessage("## Goal\nTest summary")]);
|
||||
|
||||
expect(
|
||||
getOrThrow(
|
||||
await generateSummary(
|
||||
messages,
|
||||
models,
|
||||
model,
|
||||
2000,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
),
|
||||
).toBe("## Goal\nTest summary");
|
||||
});
|
||||
|
||||
it("returns error results for failed or aborted summary generations", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const { faux: errorFaux, model: errorModel } = createFauxModel(false);
|
||||
errorFaux.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "boom" })]);
|
||||
const errorResult = await generateSummary(
|
||||
messages,
|
||||
models,
|
||||
errorModel,
|
||||
2000,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(errorResult).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "summarization_failed", message: "Summarization failed: boom" },
|
||||
});
|
||||
|
||||
const { faux: abortedFaux, model: abortedModel } = createFauxModel(false);
|
||||
abortedFaux.setResponses([fauxAssistantMessage("", { stopReason: "aborted", errorMessage: "stopped" })]);
|
||||
const abortedResult = await generateSummary(
|
||||
messages,
|
||||
models,
|
||||
abortedModel,
|
||||
2000,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(abortedResult).toMatchObject({ ok: false, error: { code: "aborted", message: "stopped" } });
|
||||
});
|
||||
|
||||
it("clamps compaction summary maxTokens to the model output cap", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const seenOptions: Array<Record<string, unknown> | undefined> = [];
|
||||
const { faux, model } = createFauxModel(false, 128000);
|
||||
faux.setResponses([
|
||||
(_context, options) => {
|
||||
seenOptions.push(options as Record<string, unknown> | undefined);
|
||||
return fauxAssistantMessage("## Goal\nTest summary");
|
||||
},
|
||||
(_context, options) => {
|
||||
seenOptions.push(options as Record<string, unknown> | undefined);
|
||||
return fauxAssistantMessage("## Goal\nTest summary");
|
||||
},
|
||||
]);
|
||||
const preparation: CompactionPreparation = {
|
||||
messagesToSummarize: messages,
|
||||
turnPrefixMessages: messages,
|
||||
retainedTail: messages,
|
||||
isSplitTurn: true,
|
||||
tokensBefore: 600000,
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
settings: { enabled: true, reserveTokens: 500000, keepRecentTokens: 20000 },
|
||||
};
|
||||
|
||||
getOrThrow(
|
||||
await compact(preparation, models, model, undefined, undefined, undefined, undefined, BACKGROUND_CONTEXT),
|
||||
);
|
||||
|
||||
expect(seenOptions.map((options) => options?.maxTokens)).toEqual([128000, 128000]);
|
||||
expect(seenOptions.map((options) => options?.cacheRetention)).toEqual(["none", "none"]);
|
||||
const sessionIds = seenOptions.map((options) => options?.sessionId);
|
||||
expect(sessionIds[0]).not.toBe(sessionIds[1]);
|
||||
});
|
||||
|
||||
it("retains per-request retries for non-harness compaction callers", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const preparation: CompactionPreparation = {
|
||||
messagesToSummarize: messages,
|
||||
turnPrefixMessages: [],
|
||||
retainedTail: messages,
|
||||
isSplitTurn: false,
|
||||
tokensBefore: 100,
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
settings: { enabled: true, reserveTokens: 2_000, keepRecentTokens: 20 },
|
||||
};
|
||||
const { faux, model } = createFauxModel(false);
|
||||
faux.setResponses([
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "rate limit exceeded" }),
|
||||
fauxAssistantMessage("recovered summary"),
|
||||
]);
|
||||
|
||||
expect(
|
||||
getOrThrow(
|
||||
await compact(
|
||||
preparation,
|
||||
models,
|
||||
model,
|
||||
undefined,
|
||||
undefined,
|
||||
{ enabled: true, maxRetries: 1, baseDelayMs: 0 },
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).summary,
|
||||
).toContain("recovered summary");
|
||||
});
|
||||
|
||||
it("returns compaction error results without throwing", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const preparation: CompactionPreparation = {
|
||||
messagesToSummarize: messages,
|
||||
turnPrefixMessages: [],
|
||||
retainedTail: messages,
|
||||
isSplitTurn: false,
|
||||
tokensBefore: 100,
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
|
||||
};
|
||||
const { faux: historyFaux, model: historyModel } = createFauxModel(false);
|
||||
historyFaux.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "history failed" })]);
|
||||
expect(
|
||||
await compact(
|
||||
preparation,
|
||||
models,
|
||||
historyModel,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "summarization_failed", message: "Summarization failed: history failed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("combines usage for split-turn compaction summaries", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const { model } = createFauxModel(false);
|
||||
const historyUsage = createMockUsage(1, 2, 3, 4);
|
||||
const turnPrefixUsage = createMockUsage(5, 6, 7, 8);
|
||||
const usageModels = createModelsWithSimpleResponses([
|
||||
{ ...fauxAssistantMessage("history summary"), usage: historyUsage },
|
||||
{ ...fauxAssistantMessage("turn prefix summary"), usage: turnPrefixUsage },
|
||||
]);
|
||||
const preparation: CompactionPreparation = {
|
||||
messagesToSummarize: messages,
|
||||
turnPrefixMessages: messages,
|
||||
isSplitTurn: true,
|
||||
tokensBefore: 100,
|
||||
retainedTail: messages,
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
|
||||
};
|
||||
|
||||
const result = getOrThrow(
|
||||
await compact(preparation, usageModels, model, undefined, undefined, undefined, undefined, BACKGROUND_CONTEXT),
|
||||
);
|
||||
|
||||
expect(result.usage).toEqual(createMockUsage(6, 8, 10, 12));
|
||||
});
|
||||
|
||||
it("passes reasoning through turn-prefix summaries when enabled", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const seenOptions: Array<Record<string, unknown> | undefined> = [];
|
||||
const { faux, model } = createFauxModel(true);
|
||||
faux.setResponses([
|
||||
(_context, options) => {
|
||||
seenOptions.push(options as Record<string, unknown> | undefined);
|
||||
return fauxAssistantMessage("## Original Request\nTest summary");
|
||||
},
|
||||
]);
|
||||
const preparation: CompactionPreparation = {
|
||||
messagesToSummarize: [],
|
||||
turnPrefixMessages: messages,
|
||||
retainedTail: messages,
|
||||
isSplitTurn: true,
|
||||
tokensBefore: 100,
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
|
||||
};
|
||||
|
||||
getOrThrow(
|
||||
await compact(preparation, models, model, undefined, "high", undefined, undefined, BACKGROUND_CONTEXT),
|
||||
);
|
||||
|
||||
expect(seenOptions[0]).toMatchObject({ reasoning: "high" });
|
||||
});
|
||||
|
||||
it("returns turn-prefix compaction errors without throwing", async () => {
|
||||
const messages: AgentMessage[] = [createUserMessage("Summarize this.")];
|
||||
const preparation: CompactionPreparation = {
|
||||
messagesToSummarize: [],
|
||||
turnPrefixMessages: messages,
|
||||
retainedTail: messages,
|
||||
isSplitTurn: true,
|
||||
tokensBefore: 100,
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
settings: { enabled: true, reserveTokens: 2000, keepRecentTokens: 20 },
|
||||
};
|
||||
const { faux, model } = createFauxModel(false);
|
||||
faux.setResponses([fauxAssistantMessage("", { stopReason: "error", errorMessage: "prefix failed" })]);
|
||||
|
||||
expect(
|
||||
await compact(preparation, models, model, undefined, undefined, undefined, undefined, BACKGROUND_CONTEXT),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "summarization_failed", message: "Turn prefix summarization failed: prefix failed" },
|
||||
});
|
||||
|
||||
const { faux: abortedFaux, model: abortedModel } = createFauxModel(false);
|
||||
abortedFaux.setResponses([fauxAssistantMessage("", { stopReason: "aborted", errorMessage: "prefix stopped" })]);
|
||||
expect(
|
||||
await compact(
|
||||
preparation,
|
||||
models,
|
||||
abortedModel,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "aborted", message: "prefix stopped" },
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a compaction result with file details", async () => {
|
||||
const u1 = createMessageEntry(createUserMessage("read a file"));
|
||||
const assistantMessage: AssistantMessage = {
|
||||
...createAssistantMessage("calling tool", createMockUsage(1000, 200)),
|
||||
content: [{ type: "toolCall", id: "tool-1", name: "read", arguments: { path: "src/index.ts" } }],
|
||||
};
|
||||
const a1 = createMessageEntry(assistantMessage, u1.id);
|
||||
const u2 = createMessageEntry(createUserMessage("continue"), a1.id);
|
||||
const a2 = createMessageEntry(createAssistantMessage("done", createMockUsage(4000, 500)), u2.id);
|
||||
const preparation = getOrThrow(prepareCompaction([u1, a1, u2, a2], DEFAULT_COMPACTION_SETTINGS));
|
||||
expect(preparation).toBeDefined();
|
||||
const { faux, model } = createFauxModel(false);
|
||||
faux.setResponses([fauxAssistantMessage("## Goal\nTest summary")]);
|
||||
const result = getOrThrow(
|
||||
await compact(preparation!, models, model, undefined, undefined, undefined, undefined, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(result.summary.length).toBeGreaterThan(0);
|
||||
expect(result.usage?.totalTokens).toBeGreaterThan(0);
|
||||
expect(result.retainedTail?.length).toBeGreaterThan(0);
|
||||
expect(result.details).toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
function convertMessages(messages: Message[]): Message[] {
|
||||
return messages;
|
||||
}
|
||||
29
third_party/pi/packages/agent/test/harness/context.test.ts
vendored
Normal file
29
third_party/pi/packages/agent/test/harness/context.test.ts
vendored
Normal file
@@ -0,0 +1,29 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
BACKGROUND_CONTEXT,
|
||||
getTelemetryContext,
|
||||
TODO_CONTEXT,
|
||||
withTelemetryContext,
|
||||
} from "../../src/harness/context.ts";
|
||||
import { InMemoryTelemetryContext, NOOP_TELEMETRY_CONTEXT } from "../../src/index.ts";
|
||||
|
||||
describe("harness Context telemetry", () => {
|
||||
it("uses no-op telemetry when none is attached", () => {
|
||||
expect(getTelemetryContext(BACKGROUND_CONTEXT)).toBe(NOOP_TELEMETRY_CONTEXT);
|
||||
expect(getTelemetryContext(TODO_CONTEXT)).toBe(NOOP_TELEMETRY_CONTEXT);
|
||||
});
|
||||
|
||||
it("carries telemetry as an ordinary context value", async () => {
|
||||
const telemetry = new InMemoryTelemetryContext();
|
||||
const context = withTelemetryContext(telemetry, BACKGROUND_CONTEXT);
|
||||
|
||||
await getTelemetryContext(context).startSpan({ name: "parent" }, async (span) => {
|
||||
const childContext = withTelemetryContext(span, context);
|
||||
await getTelemetryContext(childContext).startSpan({ name: "child" }, () => undefined);
|
||||
});
|
||||
|
||||
const spans = telemetry.getSpans();
|
||||
expect(spans.map((span) => span.name)).toEqual(["parent", "child"]);
|
||||
expect(spans[1]?.parentId).toBe(spans[0]?.id);
|
||||
});
|
||||
});
|
||||
343
third_party/pi/packages/agent/test/harness/execution-assistant.test.ts
vendored
Normal file
343
third_party/pi/packages/agent/test/harness/execution-assistant.test.ts
vendored
Normal file
@@ -0,0 +1,343 @@
|
||||
import {
|
||||
type Api,
|
||||
type AssistantMessage,
|
||||
createAssistantMessageEventStream,
|
||||
fauxAssistantMessage,
|
||||
fauxProvider,
|
||||
type Message,
|
||||
type Model,
|
||||
type SimpleStreamOptions,
|
||||
type UserMessage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { NOOP_TELEMETRY_CONTEXT } from "@earendil-works/pi-telemetry";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT, withAbortSignal } from "../../src/harness/context.ts";
|
||||
import { type AssistantResponseMetadata, streamHarnessAssistant } from "../../src/harness/execution/assistant.ts";
|
||||
import { AbortRequested } from "../../src/harness/execution/effect-gate.ts";
|
||||
import type { AgentMessage } from "../../src/types.ts";
|
||||
|
||||
function usage() {
|
||||
return {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 3,
|
||||
cacheWrite: 4,
|
||||
totalTokens: 10,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
}
|
||||
|
||||
function model(): Model<Api> {
|
||||
return {
|
||||
id: "model",
|
||||
name: "Model",
|
||||
api: "test",
|
||||
provider: "provider",
|
||||
baseUrl: "https://example.invalid",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 128_000,
|
||||
maxTokens: 16_384,
|
||||
};
|
||||
}
|
||||
|
||||
function user(text: string): UserMessage {
|
||||
return { role: "user", content: text, timestamp: 1 };
|
||||
}
|
||||
|
||||
function assistant(
|
||||
text: string,
|
||||
stopReason: Exclude<AssistantMessage["stopReason"], "pending"> = "stop",
|
||||
): AssistantMessage & { stopReason: Exclude<AssistantMessage["stopReason"], "pending"> } {
|
||||
return {
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text }],
|
||||
api: "test",
|
||||
provider: "provider",
|
||||
model: "model",
|
||||
usage: usage(),
|
||||
stopReason,
|
||||
...(stopReason === "error" ? { errorMessage: text } : {}),
|
||||
timestamp: 2,
|
||||
};
|
||||
}
|
||||
|
||||
function toProviderMessages(messages: AgentMessage[]): Message[] {
|
||||
return messages.filter(
|
||||
(message): message is Message =>
|
||||
message.role === "user" || message.role === "assistant" || message.role === "toolResult",
|
||||
);
|
||||
}
|
||||
|
||||
describe("streamHarnessAssistant", () => {
|
||||
it("maps curated options and runs the assistant lifecycle without mutating input", async () => {
|
||||
const input: AgentMessage[] = [user("original")];
|
||||
const originalArray = input;
|
||||
const requestModel = model();
|
||||
const controller = new AbortController();
|
||||
const order: string[] = [];
|
||||
const starts: AssistantMessage[] = [];
|
||||
const updates: AssistantMessage[] = [];
|
||||
const ends: AssistantMessage[] = [];
|
||||
let transformedInputWasCopy = false;
|
||||
let convertedMessages: AgentMessage[] = [];
|
||||
let requestContext: Parameters<NonNullable<SimpleStreamOptions["onPayload"]>>[0] | undefined;
|
||||
let receivedContext: { systemPrompt?: string; messages: Message[]; tools?: unknown[] } | undefined;
|
||||
let receivedOptions: SimpleStreamOptions | undefined;
|
||||
let responseMetadata: AssistantResponseMetadata | undefined;
|
||||
let startEventType: string | undefined;
|
||||
|
||||
const result = await streamHarnessAssistant(
|
||||
input,
|
||||
{
|
||||
model: requestModel,
|
||||
systemPrompt: "system",
|
||||
thinkingLevel: "high",
|
||||
streamOptions: {
|
||||
transport: "websocket",
|
||||
timeoutMs: 123,
|
||||
maxRetries: 2,
|
||||
maxRetryDelayMs: 456,
|
||||
headers: { authorization: "test" },
|
||||
metadata: { tenant: "one" },
|
||||
cacheRetention: "long",
|
||||
deferred: { window: "1h" },
|
||||
},
|
||||
transformContext: async (context) => {
|
||||
order.push("transform_context");
|
||||
transformedInputWasCopy = context.messages !== originalArray;
|
||||
context.messages.push(user("injected"));
|
||||
return { messages: context.messages, systemPrompt: "transformed system" };
|
||||
},
|
||||
toProviderMessages: (messages) => {
|
||||
order.push("to_provider_messages");
|
||||
convertedMessages = messages;
|
||||
return toProviderMessages(messages);
|
||||
},
|
||||
beforePayload: (payload, seenModel) => {
|
||||
order.push("before_payload");
|
||||
expect(seenModel.id).toBe("resolved");
|
||||
requestContext = payload;
|
||||
return { replaced: true };
|
||||
},
|
||||
afterResponse: async (message, metadata) => {
|
||||
order.push("after_response");
|
||||
responseMetadata = metadata;
|
||||
return { ...message, content: [{ type: "text", text: "transformed" }] };
|
||||
},
|
||||
request: async (context, options) => {
|
||||
order.push("request");
|
||||
receivedContext = context;
|
||||
receivedOptions = options;
|
||||
expect(await options.onPayload?.({ original: true }, { ...requestModel, id: "resolved" })).toEqual({
|
||||
replaced: true,
|
||||
});
|
||||
await options.onResponse?.({ status: 201, headers: { "request-id": "r1" } }, requestModel);
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
const initial = { ...assistant(""), stopReason: "pending" as const };
|
||||
const partial = { ...initial, content: [{ type: "text" as const, text: "raw" }] };
|
||||
stream.push({ type: "start", partial: initial });
|
||||
stream.push({ type: "text_delta", contentIndex: 0, delta: "raw", partial });
|
||||
stream.push({ type: "done", reason: "stop", message: assistant("raw") });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
observer: {
|
||||
start(message, event) {
|
||||
order.push("observer_start");
|
||||
starts.push(message);
|
||||
startEventType = event.type;
|
||||
},
|
||||
update(message) {
|
||||
order.push("observer_update");
|
||||
updates.push(message);
|
||||
},
|
||||
end(message) {
|
||||
order.push("observer_end");
|
||||
ends.push(message);
|
||||
},
|
||||
},
|
||||
},
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
);
|
||||
|
||||
expect(input).toEqual([user("original")]);
|
||||
expect(transformedInputWasCopy).toBe(true);
|
||||
expect(convertedMessages).toEqual([user("original"), user("injected")]);
|
||||
expect(receivedContext).toMatchObject({
|
||||
systemPrompt: "transformed system",
|
||||
messages: [user("original"), user("injected")],
|
||||
});
|
||||
expect(requestContext).toEqual({ original: true });
|
||||
expect(receivedOptions).toMatchObject({
|
||||
transport: "websocket",
|
||||
timeoutMs: 123,
|
||||
maxRetries: 2,
|
||||
maxRetryDelayMs: 456,
|
||||
headers: { authorization: "test" },
|
||||
metadata: { tenant: "one" },
|
||||
cacheRetention: "long",
|
||||
deferred: { window: "1h" },
|
||||
reasoning: "high",
|
||||
signal: controller.signal,
|
||||
telemetryContext: NOOP_TELEMETRY_CONTEXT,
|
||||
});
|
||||
expect(responseMetadata).toEqual({ status: 201, headers: { "request-id": "r1" } });
|
||||
expect(starts).toHaveLength(1);
|
||||
expect(startEventType).toBe("start");
|
||||
expect(starts[0]).not.toBe(updates[0]);
|
||||
expect(updates).toHaveLength(1);
|
||||
expect(ends[0]).toBe(result);
|
||||
expect(result.content).toEqual([{ type: "text", text: "transformed" }]);
|
||||
expect(order).toEqual([
|
||||
"transform_context",
|
||||
"to_provider_messages",
|
||||
"request",
|
||||
"before_payload",
|
||||
"observer_start",
|
||||
"observer_update",
|
||||
"after_response",
|
||||
"observer_end",
|
||||
]);
|
||||
});
|
||||
|
||||
it("runs against the faux provider request boundary", async () => {
|
||||
const faux = fauxProvider({ tokenSize: { min: 1, max: 1 } });
|
||||
faux.setResponses([fauxAssistantMessage("hello")]);
|
||||
const lifecycle: string[] = [];
|
||||
let seenContext: Message[] = [];
|
||||
|
||||
const result = await streamHarnessAssistant(
|
||||
[user("prompt")],
|
||||
{
|
||||
model: faux.getModel(),
|
||||
systemPrompt: "system",
|
||||
thinkingLevel: "off",
|
||||
streamOptions: {},
|
||||
toProviderMessages,
|
||||
request: (context, options) => {
|
||||
seenContext = context.messages;
|
||||
return faux.provider.streamSimple(faux.getModel(), context, options);
|
||||
},
|
||||
observer: {
|
||||
start() {
|
||||
lifecycle.push("start");
|
||||
},
|
||||
update() {
|
||||
lifecycle.push("update");
|
||||
},
|
||||
end() {
|
||||
lifecycle.push("end");
|
||||
},
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(seenContext).toEqual([user("prompt")]);
|
||||
expect(result.content).toEqual([{ type: "text", text: "hello" }]);
|
||||
expect(lifecycle[0]).toBe("start");
|
||||
expect(lifecycle.at(-1)).toBe("end");
|
||||
expect(lifecycle.filter((event) => event === "update").length).toBeGreaterThan(0);
|
||||
});
|
||||
|
||||
it("rejects a successful terminal event before start", async () => {
|
||||
const final = assistant("complete");
|
||||
let options: SimpleStreamOptions | undefined;
|
||||
await expect(
|
||||
streamHarnessAssistant(
|
||||
[user("prompt")],
|
||||
{
|
||||
model: model(),
|
||||
systemPrompt: "system",
|
||||
thinkingLevel: "off",
|
||||
streamOptions: {},
|
||||
toProviderMessages,
|
||||
request: (_context, requestOptions) => {
|
||||
options = requestOptions;
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => stream.push({ type: "done", reason: "stop", message: final }));
|
||||
return stream;
|
||||
},
|
||||
observer: { start() {}, update() {}, end() {} },
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow("done before start");
|
||||
expect(options).not.toHaveProperty("reasoning");
|
||||
});
|
||||
|
||||
it("keeps the raw settlement when cancellation interrupts after_response", async () => {
|
||||
const final = assistant("raw");
|
||||
const ended: AssistantMessage[] = [];
|
||||
const result = await streamHarnessAssistant(
|
||||
[user("prompt")],
|
||||
{
|
||||
model: model(),
|
||||
systemPrompt: "system",
|
||||
thinkingLevel: "off",
|
||||
streamOptions: {},
|
||||
toProviderMessages,
|
||||
request: () => {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "start", partial: { ...final, content: [], stopReason: "pending" } });
|
||||
stream.push({ type: "done", reason: "stop", message: final });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
afterResponse: async () => {
|
||||
throw new AbortRequested(Promise.resolve());
|
||||
},
|
||||
observer: {
|
||||
start() {},
|
||||
update() {},
|
||||
end(message) {
|
||||
ended.push(message);
|
||||
},
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(result).toBe(final);
|
||||
expect(ended).toEqual([final]);
|
||||
});
|
||||
|
||||
it("returns provider error settlements through the same lifecycle", async () => {
|
||||
const final = assistant("provider failed", "error");
|
||||
const events: string[] = [];
|
||||
const result = await streamHarnessAssistant(
|
||||
[user("prompt")],
|
||||
{
|
||||
model: model(),
|
||||
systemPrompt: "system",
|
||||
thinkingLevel: "off",
|
||||
streamOptions: {},
|
||||
toProviderMessages,
|
||||
request: () => {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
queueMicrotask(() => stream.push({ type: "error", reason: "error", error: final }));
|
||||
return stream;
|
||||
},
|
||||
observer: {
|
||||
start() {
|
||||
throw new Error("pre-generation error must not synthesize start");
|
||||
},
|
||||
update() {
|
||||
events.push("update");
|
||||
},
|
||||
end(message) {
|
||||
events.push(`end:${message.stopReason}`);
|
||||
},
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(result).toBe(final);
|
||||
expect(events).toEqual(["end:error"]);
|
||||
});
|
||||
});
|
||||
670
third_party/pi/packages/agent/test/harness/execution-primitives.test.ts
vendored
Normal file
670
third_party/pi/packages/agent/test/harness/execution-primitives.test.ts
vendored
Normal file
@@ -0,0 +1,670 @@
|
||||
import { fauxProvider } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import {
|
||||
BACKGROUND_CONTEXT,
|
||||
createContextKey,
|
||||
getTelemetryContext,
|
||||
withAbortSignal,
|
||||
withContextValue,
|
||||
withTelemetryContext,
|
||||
} from "../../src/harness/context.ts";
|
||||
import { HarnessEventBus } from "../../src/harness/events.ts";
|
||||
import { AbortRequested, createGate } from "../../src/harness/execution/effect-gate.ts";
|
||||
import { HookRegistry } from "../../src/harness/hooks.ts";
|
||||
import { InMemoryTelemetryContext } from "../../src/index.ts";
|
||||
|
||||
function deferred(): { promise: Promise<void>; resolve(): void } {
|
||||
let resolvePromise: (() => void) | undefined;
|
||||
const promise = new Promise<void>((resolve) => {
|
||||
resolvePromise = resolve;
|
||||
});
|
||||
return { promise, resolve: () => resolvePromise?.() };
|
||||
}
|
||||
|
||||
describe("createGate", () => {
|
||||
it("closes starts synchronously and signals only after cancellation commits", async () => {
|
||||
const { gate, control } = createGate();
|
||||
const cancellation = deferred();
|
||||
control.beginAbort(cancellation.promise);
|
||||
|
||||
let refusal: unknown;
|
||||
try {
|
||||
gate.admit(() => undefined);
|
||||
} catch (error) {
|
||||
refusal = error;
|
||||
}
|
||||
expect(refusal).toBeInstanceOf(AbortRequested);
|
||||
expect((refusal as AbortRequested).cancellation).toBe(cancellation.promise);
|
||||
expect(gate.signal.aborted).toBe(false);
|
||||
cancellation.resolve();
|
||||
await cancellation.promise;
|
||||
control.signalAbort();
|
||||
expect(gate.signal.aborted).toBe(true);
|
||||
});
|
||||
|
||||
it("permanently closes and signals admitted work", () => {
|
||||
const { gate, control } = createGate();
|
||||
const error = new Error("closed");
|
||||
control.close(error);
|
||||
expect(() => gate.admit(() => undefined)).toThrow(error);
|
||||
expect(gate.signal.aborted).toBe(true);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HookRegistry", () => {
|
||||
it("aggregates before_run messages in registration order with each handler seeing prior output", async () => {
|
||||
const errors: Error[] = [];
|
||||
const hooks = new HookRegistry((error) => {
|
||||
errors.push(error);
|
||||
});
|
||||
const second = vi.fn((event: { prompt: Array<{ role: string }> }) => {
|
||||
expect(event.prompt).toHaveLength(2);
|
||||
return { messages: [{ role: "user" as const, content: "second", timestamp: 3 }] };
|
||||
});
|
||||
hooks.on("before_run", () => ({ messages: [{ role: "user", content: "first", timestamp: 2 }] }));
|
||||
hooks.on("before_run", second);
|
||||
|
||||
const result = await hooks.runWithGate(
|
||||
"before_run",
|
||||
{
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
prompt: [{ role: "user", content: "prompt", timestamp: 1 }],
|
||||
resources: {},
|
||||
},
|
||||
createGate().gate,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
messages: [
|
||||
{ role: "user", content: "first" },
|
||||
{ role: "user", content: "second" },
|
||||
],
|
||||
});
|
||||
expect(second).toHaveBeenCalledOnce();
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
|
||||
it("checks the effect gate immediately before the complete before_run pipeline", async () => {
|
||||
const hooks = new HookRegistry(() => {});
|
||||
const calls: string[] = [];
|
||||
const release = deferred();
|
||||
hooks.on("before_run", async () => {
|
||||
calls.push("first:start");
|
||||
await release.promise;
|
||||
calls.push("first:end");
|
||||
return undefined;
|
||||
});
|
||||
hooks.on("before_run", () => {
|
||||
calls.push("second");
|
||||
return undefined;
|
||||
});
|
||||
const event = { lane: "main", runId: "run", prompt: [], resources: {} };
|
||||
const closed = new Error("closed");
|
||||
const closedGate = createGate();
|
||||
closedGate.control.close(closed);
|
||||
expect(() => hooks.runWithGate("before_run", event, closedGate.gate, BACKGROUND_CONTEXT)).toThrow(closed);
|
||||
expect(calls).toEqual([]);
|
||||
|
||||
const running = hooks.runWithGate("before_run", event, createGate().gate, BACKGROUND_CONTEXT);
|
||||
expect(calls).toEqual(["first:start"]);
|
||||
hooks.on("before_run", () => {
|
||||
calls.push("late");
|
||||
return undefined;
|
||||
});
|
||||
hooks.close(closed);
|
||||
release.resolve();
|
||||
await running;
|
||||
expect(calls).toEqual(["first:start", "first:end", "second"]);
|
||||
});
|
||||
|
||||
it("rejects pre-aborted invocations and combines admitted hook cancellation with the effect gate", async () => {
|
||||
const event = { operation: "run" as const, lane: "main", runId: "run" };
|
||||
const preAbortedHooks = new HookRegistry(() => {});
|
||||
const handler = vi.fn();
|
||||
preAbortedHooks.on("before_drive", handler);
|
||||
const preAbortedGate = createGate();
|
||||
const invocationController = new AbortController();
|
||||
const abortReason = new Error("invocation aborted");
|
||||
invocationController.abort(abortReason);
|
||||
|
||||
expect(() =>
|
||||
preAbortedHooks.runWithGate(
|
||||
"before_drive",
|
||||
event,
|
||||
preAbortedGate.gate,
|
||||
withAbortSignal(invocationController.signal, BACKGROUND_CONTEXT),
|
||||
),
|
||||
).toThrow(abortReason);
|
||||
expect(handler).not.toHaveBeenCalled();
|
||||
expect(() => preAbortedGate.gate.admit(() => undefined)).not.toThrow();
|
||||
|
||||
const admittedHooks = new HookRegistry(() => {});
|
||||
const admittedGate = createGate();
|
||||
let admittedSignal: AbortSignal | undefined;
|
||||
admittedHooks.on("before_drive", (_event, context) => {
|
||||
admittedSignal = context.abortSignal;
|
||||
});
|
||||
await admittedHooks.runWithGate("before_drive", event, admittedGate.gate, BACKGROUND_CONTEXT);
|
||||
expect(admittedSignal?.aborted).toBe(false);
|
||||
|
||||
const gateReason = new Error("gate closed");
|
||||
admittedGate.control.close(gateReason);
|
||||
expect(admittedSignal?.aborted).toBe(true);
|
||||
expect(admittedSignal?.reason).toBe(gateReason);
|
||||
});
|
||||
|
||||
it("passes tool handlers a child context of the active hook span", async () => {
|
||||
const telemetry = new InMemoryTelemetryContext();
|
||||
const valueKey = createContextKey<string>("hook.test.value");
|
||||
const hooks = new HookRegistry(() => {});
|
||||
let receivedValue: string | undefined;
|
||||
hooks.on("before_tool", async (_event, context) => {
|
||||
receivedValue = context.value(valueKey);
|
||||
await getTelemetryContext(context).startSpan({ name: "handler.child" }, () => undefined);
|
||||
return undefined;
|
||||
});
|
||||
|
||||
await telemetry.startSpan({ name: "invocation" }, (invocationSpan) =>
|
||||
hooks.runToolWithGate(
|
||||
"before_tool",
|
||||
{
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
toolCallId: "call",
|
||||
toolName: "tool",
|
||||
args: {},
|
||||
},
|
||||
createGate().gate,
|
||||
withContextValue(valueKey, "preserved", withTelemetryContext(invocationSpan, BACKGROUND_CONTEXT)),
|
||||
),
|
||||
);
|
||||
|
||||
expect(receivedValue).toBe("preserved");
|
||||
const spans = telemetry.getSpans();
|
||||
const invocationSpan = spans.find((span) => span.name === "invocation");
|
||||
const hookSpan = spans.find((span) => span.name === "pi.harness.hook");
|
||||
const handlerSpan = spans.find((span) => span.name === "handler.child");
|
||||
expect(hookSpan?.parentId).toBe(invocationSpan?.id);
|
||||
expect(handlerSpan?.parentId).toBe(hookSpan?.id);
|
||||
});
|
||||
|
||||
it("treats registration ids as optional metadata and fails before_drive closed", async () => {
|
||||
const errors: Error[] = [];
|
||||
const hooks = new HookRegistry((error) => {
|
||||
errors.push(error);
|
||||
});
|
||||
const failure = new Error("prerequisite failed");
|
||||
const later = vi.fn();
|
||||
hooks.on(
|
||||
"before_drive",
|
||||
() => {
|
||||
throw failure;
|
||||
},
|
||||
{ id: "duplicate" },
|
||||
);
|
||||
hooks.on("before_drive", later, { id: "duplicate" });
|
||||
|
||||
await expect(
|
||||
hooks.runWithGate(
|
||||
"before_drive",
|
||||
{ lane: "main", runId: "run", operation: "run" },
|
||||
createGate().gate,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toBe(failure);
|
||||
expect(later).not.toHaveBeenCalled();
|
||||
expect(errors).toEqual([failure]);
|
||||
});
|
||||
|
||||
it("chains transform_context output and isolates handler failures", async () => {
|
||||
const errors: Error[] = [];
|
||||
const hooks = new HookRegistry((error) => {
|
||||
errors.push(error);
|
||||
});
|
||||
const messages = [{ role: "user" as const, content: "transformed", timestamp: 2 }];
|
||||
hooks.on("transform_context", () => ({ messages, systemPrompt: "first" }));
|
||||
hooks.on("transform_context", (event) => {
|
||||
expect(event.messages).toBe(messages);
|
||||
expect(event.systemPrompt).toBe("first");
|
||||
throw new Error("ignored transform failure");
|
||||
});
|
||||
hooks.on("transform_context", (event) => {
|
||||
expect(event.messages).toBe(messages);
|
||||
expect(event.systemPrompt).toBe("first");
|
||||
return { systemPrompt: "final" };
|
||||
});
|
||||
|
||||
const result = await hooks.runWithGate(
|
||||
"transform_context",
|
||||
{
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
messages: [{ role: "user", content: "original", timestamp: 1 }],
|
||||
systemPrompt: "base",
|
||||
},
|
||||
createGate().gate,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(result).toEqual({ messages, systemPrompt: "final" });
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0]?.message).toBe("ignored transform failure");
|
||||
});
|
||||
|
||||
it("preserves clear-all before_request patches across later handlers", async () => {
|
||||
const hooks = new HookRegistry(() => {});
|
||||
hooks.on("before_request", () => ({ streamOptions: { headers: undefined, metadata: undefined } }));
|
||||
hooks.on("before_request", (event) => {
|
||||
expect(event.streamOptions.headers).toBeUndefined();
|
||||
expect(event.streamOptions.metadata).toBeUndefined();
|
||||
return { streamOptions: { headers: { c: "3" }, metadata: { y: 2 } } };
|
||||
});
|
||||
const result = await hooks.runWithGate(
|
||||
"before_request",
|
||||
{
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
model: fauxProvider().getModel(),
|
||||
step: "assistant",
|
||||
attempt: 1,
|
||||
streamOptions: { headers: { a: "1", b: "2" }, metadata: { x: 1 } },
|
||||
},
|
||||
createGate().gate,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result).toEqual({
|
||||
streamOptions: {
|
||||
headers: { a: undefined, b: undefined, c: "3" },
|
||||
metadata: { x: undefined, y: 2 },
|
||||
},
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves earlier after_tool fields when a later patch returns undefined", async () => {
|
||||
const hooks = new HookRegistry(() => {});
|
||||
hooks.on("after_tool", () => ({ content: [{ type: "text", text: "patched" }] }));
|
||||
hooks.on("after_tool", () => ({ content: undefined, isError: false }));
|
||||
const result = await hooks.runWithGate(
|
||||
"after_tool",
|
||||
{
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
toolCallId: "call",
|
||||
toolName: "tool",
|
||||
args: {},
|
||||
content: [{ type: "text", text: "raw" }],
|
||||
isError: true,
|
||||
},
|
||||
createGate().gate,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result).toEqual({ content: [{ type: "text", text: "patched" }], isError: false });
|
||||
});
|
||||
|
||||
it("accepts explicit false structural declines and rejects true conflicts", async () => {
|
||||
const errors: Error[] = [];
|
||||
const hooks = new HookRegistry((error) => {
|
||||
errors.push(error);
|
||||
});
|
||||
const ignored = { summary: "ignored", readFiles: [], modifiedFiles: [] };
|
||||
const selected = { summary: "selected", readFiles: [], modifiedFiles: [] };
|
||||
hooks.on("before_navigation", () => ({ decline: true, summary: ignored }));
|
||||
hooks.on("before_navigation", () => ({ decline: false, summary: selected }));
|
||||
|
||||
const result = await hooks.runWithGate(
|
||||
"before_navigation",
|
||||
{
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
targetId: "target",
|
||||
preparation: {
|
||||
messages: [],
|
||||
fileOps: { read: new Set(), written: new Set(), edited: new Set() },
|
||||
totalTokens: 0,
|
||||
},
|
||||
},
|
||||
createGate().gate,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result).toEqual({ decline: false, summary: selected });
|
||||
expect(errors).toHaveLength(1);
|
||||
expect(errors[0].message).toMatch(/cannot return both decline and summary/);
|
||||
});
|
||||
|
||||
it("admits the complete before_drive pipeline as one gated effect", async () => {
|
||||
const hooks = new HookRegistry(() => {});
|
||||
const calls: string[] = [];
|
||||
const release = deferred();
|
||||
hooks.on("before_drive", async () => {
|
||||
calls.push("first:start");
|
||||
await release.promise;
|
||||
calls.push("first:end");
|
||||
});
|
||||
hooks.on("before_drive", () => {
|
||||
calls.push("second");
|
||||
});
|
||||
const event = { operation: "run" as const, lane: "main", runId: "run" };
|
||||
|
||||
const abortFirstGate = createGate();
|
||||
const cancellation = Promise.resolve();
|
||||
abortFirstGate.control.beginAbort(cancellation);
|
||||
expect(() => hooks.runWithGate("before_drive", event, abortFirstGate.gate, BACKGROUND_CONTEXT)).toThrow(
|
||||
AbortRequested,
|
||||
);
|
||||
expect(calls).toEqual([]);
|
||||
|
||||
const startFirstGate = createGate();
|
||||
const running = hooks.runWithGate("before_drive", event, startFirstGate.gate, BACKGROUND_CONTEXT);
|
||||
expect(calls).toEqual(["first:start"]);
|
||||
startFirstGate.control.beginAbort(cancellation);
|
||||
startFirstGate.control.signalAbort();
|
||||
release.resolve();
|
||||
await running;
|
||||
expect(calls).toEqual(["first:start", "first:end", "second"]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("HarnessEventBus", () => {
|
||||
it("buffers between snapshot and start, then delivers each event once in order", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const watcher = bus.watch({ tipId: null }, () => true, BACKGROUND_CONTEXT);
|
||||
await bus.emit({ type: "run_start", runId: "one", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
const seen: string[] = [];
|
||||
watcher.start((event) => {
|
||||
seen.push(`${event.type}:${"runId" in event ? event.runId : ""}`);
|
||||
});
|
||||
await bus.emit({ type: "run_start", runId: "two", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(watcher.snapshot).toEqual({ tipId: null });
|
||||
expect(seen).toEqual(["run_start:one", "run_start:two"]);
|
||||
watcher.unsubscribe();
|
||||
await bus.emit({ type: "run_start", runId: "three", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
expect(seen).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("drops pre-snapshot delivery and holds later events when resnapshotting inside a listener", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const listenerStarted = deferred();
|
||||
const releaseListener = deferred();
|
||||
const resnapshotDone = deferred();
|
||||
const watcher = bus.watch(
|
||||
{ version: "old" },
|
||||
() => true,
|
||||
BACKGROUND_CONTEXT,
|
||||
async (_context, markBoundary) => {
|
||||
markBoundary();
|
||||
return { version: "fresh" };
|
||||
},
|
||||
);
|
||||
const queueEvents: string[] = [];
|
||||
watcher.start(async (event, context) => {
|
||||
if (event.type === "run_start" && event.runId === "blocking") {
|
||||
listenerStarted.resolve();
|
||||
await releaseListener.promise;
|
||||
} else if (event.type === "navigation_end") {
|
||||
await watcher.resnapshot(context);
|
||||
resnapshotDone.resolve();
|
||||
} else if (event.type === "queue_update") {
|
||||
queueEvents.push(event.queues[0]?.entryId ?? "empty");
|
||||
}
|
||||
});
|
||||
await bus.emit({ type: "run_start", runId: "blocking", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
await listenerStarted.promise;
|
||||
const queued = bus.emitBatch(
|
||||
[
|
||||
{
|
||||
type: "navigation_end",
|
||||
runId: "navigation",
|
||||
status: "completed",
|
||||
fromTipId: null,
|
||||
tipId: null,
|
||||
endedAt: 2,
|
||||
lane: "main",
|
||||
},
|
||||
{
|
||||
type: "queue_update",
|
||||
queues: [
|
||||
{
|
||||
entryId: "stale",
|
||||
kind: "nextRun",
|
||||
type: "message",
|
||||
message: { role: "user", content: "stale", timestamp: 2 },
|
||||
},
|
||||
],
|
||||
lane: "main",
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
releaseListener.resolve();
|
||||
await Promise.all([queued, resnapshotDone.promise]);
|
||||
|
||||
expect(watcher.snapshot).toEqual({ version: "fresh" });
|
||||
expect(queueEvents).toEqual([]);
|
||||
await bus.emit(
|
||||
{
|
||||
type: "queue_update",
|
||||
queues: [
|
||||
{
|
||||
entryId: "later",
|
||||
kind: "nextRun",
|
||||
type: "message",
|
||||
message: { role: "user", content: "later", timestamp: 3 },
|
||||
},
|
||||
],
|
||||
lane: "main",
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await expect.poll(() => queueEvents).toEqual(["later"]);
|
||||
});
|
||||
|
||||
it("isolates each listener from payload mutation", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
let observed: string[] = [];
|
||||
bus.on("config_update", (event) => {
|
||||
if (event.property === "activeTools") event.value.push("mutated");
|
||||
});
|
||||
bus.on("config_update", (event) => {
|
||||
if (event.property === "activeTools") observed = event.value;
|
||||
});
|
||||
await bus.emit(
|
||||
{
|
||||
type: "config_update",
|
||||
property: "activeTools",
|
||||
value: ["read"],
|
||||
previous: [],
|
||||
lane: "main",
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(observed).toEqual(["read"]);
|
||||
});
|
||||
|
||||
it("serializes concurrent publications in process order", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
const seen: string[] = [];
|
||||
bus.on("run_start", async (event) => {
|
||||
seen.push(`${event.runId}:start`);
|
||||
if (event.runId === "one") {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
seen.push(`${event.runId}:end`);
|
||||
});
|
||||
|
||||
const one = bus.emit({ type: "run_start", runId: "one", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
const two = bus.emit({ type: "run_start", runId: "two", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(seen).toEqual(["one:start"]);
|
||||
release.resolve();
|
||||
await Promise.all([one, two]);
|
||||
expect(seen).toEqual(["one:start", "one:end", "two:start", "two:end"]);
|
||||
});
|
||||
|
||||
it("keeps concurrent batches contiguous with their emitting contexts", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const sourceKey = createContextKey<string>("event.batch.source");
|
||||
const firstContext = withContextValue(sourceKey, "first", BACKGROUND_CONTEXT);
|
||||
const secondContext = withContextValue(sourceKey, "second", BACKGROUND_CONTEXT);
|
||||
const seen: Array<{ runId: string; context: typeof firstContext }> = [];
|
||||
bus.on("run_start", async (event, context) => {
|
||||
seen.push({ runId: event.runId, context });
|
||||
await Promise.resolve();
|
||||
});
|
||||
|
||||
const first = bus.emitBatch(
|
||||
[
|
||||
{ type: "run_start", runId: "a1", startedAt: 1, lane: "main" },
|
||||
{ type: "run_start", runId: "a2", startedAt: 1, lane: "main" },
|
||||
],
|
||||
firstContext,
|
||||
);
|
||||
const second = bus.emitBatch(
|
||||
[
|
||||
{ type: "run_start", runId: "b1", startedAt: 1, lane: "worker" },
|
||||
{ type: "run_start", runId: "b2", startedAt: 1, lane: "worker" },
|
||||
],
|
||||
secondContext,
|
||||
);
|
||||
|
||||
await Promise.all([first, second]);
|
||||
expect(seen.map(({ runId }) => runId)).toEqual(["a1", "a2", "b1", "b2"]);
|
||||
expect(seen.slice(0, 2).every(({ context }) => context === firstContext)).toBe(true);
|
||||
expect(seen.slice(2).every(({ context }) => context === secondContext)).toBe(true);
|
||||
});
|
||||
|
||||
it("binds listeners and watchers when a batch is emitted", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
bus.on("run_start", async (event) => {
|
||||
if (event.runId !== "blocking") return;
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
});
|
||||
const blocking = bus.emit(
|
||||
{ type: "run_start", runId: "blocking", startedAt: 1, lane: "main" },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await started.promise;
|
||||
|
||||
const queued = bus.emit({ type: "run_start", runId: "queued", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
const lateListenerEvents: string[] = [];
|
||||
bus.on("run_start", (event) => {
|
||||
lateListenerEvents.push(event.runId);
|
||||
});
|
||||
const lateWatcherEvents: string[] = [];
|
||||
const lateWatcher = bus.watch({}, (event) => event.type === "run_start", BACKGROUND_CONTEXT);
|
||||
lateWatcher.start((event) => {
|
||||
if (event.type === "run_start") lateWatcherEvents.push(event.runId);
|
||||
});
|
||||
|
||||
release.resolve();
|
||||
await Promise.all([blocking, queued]);
|
||||
expect(lateListenerEvents).toEqual([]);
|
||||
expect(lateWatcherEvents).toEqual([]);
|
||||
|
||||
await bus.emit({ type: "run_start", runId: "later", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
expect(lateListenerEvents).toEqual(["later"]);
|
||||
expect(lateWatcherEvents).toEqual(["later"]);
|
||||
});
|
||||
|
||||
it("resolves an empty batch without delivery", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const listener = vi.fn();
|
||||
bus.on("run_start", listener);
|
||||
|
||||
await bus.emitBatch([], BACKGROUND_CONTEXT);
|
||||
|
||||
expect(listener).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("drains already emitted contiguous batches during close and ignores later publication", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
const seen: string[] = [];
|
||||
bus.on("run_start", async (event) => {
|
||||
seen.push(event.runId);
|
||||
if (event.runId === "blocking") {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
}
|
||||
});
|
||||
const blocking = bus.emit(
|
||||
{ type: "run_start", runId: "blocking", startedAt: 1, lane: "main" },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await started.promise;
|
||||
const batch = bus.emitBatch(
|
||||
[
|
||||
{ type: "run_start", runId: "one", startedAt: 1, lane: "main" },
|
||||
{ type: "run_start", runId: "two", startedAt: 1, lane: "main" },
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
bus.close(new Error("closed"));
|
||||
await bus.emit({ type: "run_start", runId: "late", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
release.resolve();
|
||||
await Promise.all([blocking, batch]);
|
||||
|
||||
expect(seen).toEqual(["blocking", "one", "two"]);
|
||||
});
|
||||
|
||||
it("continues watcher delivery after listener failure and reports it", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const failures: string[] = [];
|
||||
bus.on("handler_error", (event) => {
|
||||
failures.push(event.error);
|
||||
});
|
||||
const watcher = bus.watch({}, (event) => event.type === "run_start", BACKGROUND_CONTEXT);
|
||||
const seen: string[] = [];
|
||||
watcher.start((event) => {
|
||||
if (event.type !== "run_start") return;
|
||||
if (event.runId === "one") throw new Error("watcher failed");
|
||||
seen.push(event.runId);
|
||||
});
|
||||
|
||||
await bus.emit({ type: "run_start", runId: "one", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
await bus.emit({ type: "run_start", runId: "two", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(seen).toEqual(["two"]);
|
||||
expect(failures).toEqual(["watcher failed"]);
|
||||
});
|
||||
|
||||
it("isolates listener failures and emits handler_error", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const failures: string[] = [];
|
||||
bus.on("run_start", () => {
|
||||
throw new Error("listener failed");
|
||||
});
|
||||
bus.on("handler_error", (event) => {
|
||||
failures.push(event.error);
|
||||
});
|
||||
await bus.emit({ type: "run_start", runId: "run", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
expect(failures).toEqual(["listener failed"]);
|
||||
});
|
||||
|
||||
it("does not recurse when a handler_error listener fails", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
let handlerErrors = 0;
|
||||
bus.on("run_start", () => {
|
||||
throw new Error("listener failed");
|
||||
});
|
||||
bus.on("handler_error", () => {
|
||||
handlerErrors += 1;
|
||||
throw new Error("error listener failed");
|
||||
});
|
||||
|
||||
await bus.emit({ type: "run_start", runId: "run", startedAt: 1, lane: "main" }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(handlerErrors).toBe(1);
|
||||
});
|
||||
});
|
||||
247
third_party/pi/packages/agent/test/harness/execution-tools.test.ts
vendored
Normal file
247
third_party/pi/packages/agent/test/harness/execution-tools.test.ts
vendored
Normal file
@@ -0,0 +1,247 @@
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { AbortRequested, createGate, type Gate } from "../../src/harness/execution/effect-gate.ts";
|
||||
import {
|
||||
applyBeforeToolDecision,
|
||||
createToolResultMessage,
|
||||
executeToolCall,
|
||||
finalizeToolCall,
|
||||
prepareToolCall,
|
||||
} from "../../src/harness/execution/tools.ts";
|
||||
import type { AgentHarnessTool } from "../../src/harness/types.ts";
|
||||
import type { AgentToolCall, AgentToolResult } from "../../src/types.ts";
|
||||
|
||||
const parameters = Type.Object({ value: Type.String() });
|
||||
|
||||
function call(arguments_: Record<string, unknown> = { value: "input" }): AgentToolCall {
|
||||
return { type: "toolCall", id: "call-1", name: "echo", arguments: arguments_ };
|
||||
}
|
||||
|
||||
function tool(
|
||||
overrides: Partial<AgentHarnessTool<undefined, typeof parameters, { value: string }>> = {},
|
||||
): AgentHarnessTool<undefined, typeof parameters, { value: string }> {
|
||||
return {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo input",
|
||||
parameters,
|
||||
async execute(_toolCallId, args) {
|
||||
return {
|
||||
content: [{ type: "text", text: args.value }],
|
||||
details: { value: args.value },
|
||||
};
|
||||
},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
function text(result: AgentToolResult<unknown>): string {
|
||||
return result.content.flatMap((content) => (content.type === "text" ? [content.text] : [])).join("\n");
|
||||
}
|
||||
|
||||
function isImmediate(
|
||||
result: ReturnType<typeof prepareToolCall> | ReturnType<typeof applyBeforeToolDecision>,
|
||||
): result is Extract<ReturnType<typeof prepareToolCall>, { kind: "immediate" }> {
|
||||
return "kind" in result;
|
||||
}
|
||||
|
||||
function clearPrepared(callToClear: ReturnType<typeof prepareToolCall>) {
|
||||
if (isImmediate(callToClear)) throw new Error("expected prepared call");
|
||||
const cleared = applyBeforeToolDecision(callToClear, undefined);
|
||||
if (isImmediate(cleared)) throw new Error("expected cleared call");
|
||||
return cleared;
|
||||
}
|
||||
|
||||
function effectGate(): Gate {
|
||||
return createGate().gate;
|
||||
}
|
||||
|
||||
const invocation = {
|
||||
invocationId: "result-1",
|
||||
operationId: "operation-1",
|
||||
turnId: "turn-1",
|
||||
getMemo: async () => undefined,
|
||||
setMemo: async () => {},
|
||||
};
|
||||
|
||||
describe("tool execution primitives", () => {
|
||||
it("prepares arguments before validation and preserves the provider call", () => {
|
||||
const providerCall = call({ legacy: "prepared" });
|
||||
const prepared = prepareToolCall(providerCall, [
|
||||
tool({
|
||||
prepareArguments(args) {
|
||||
return { value: (args as { legacy: string }).legacy };
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
expect(isImmediate(prepared)).toBe(false);
|
||||
if (isImmediate(prepared)) return;
|
||||
expect(prepared.toolCall).toBe(providerCall);
|
||||
expect(prepared.args).toEqual({ value: "prepared" });
|
||||
expect(providerCall.arguments).toEqual({ legacy: "prepared" });
|
||||
});
|
||||
|
||||
it("returns immediate errors for unknown tools, preparation throws, and invalid arguments", () => {
|
||||
const unknown = prepareToolCall(call(), []);
|
||||
const preparationFailure = prepareToolCall(call(), [
|
||||
tool({
|
||||
prepareArguments() {
|
||||
throw new Error("cannot prepare");
|
||||
},
|
||||
}),
|
||||
]);
|
||||
const invalid = prepareToolCall(call({}), [tool()]);
|
||||
|
||||
expect(isImmediate(unknown) ? text(unknown.result) : "").toBe('Tool "echo" is unavailable');
|
||||
expect(isImmediate(unknown) ? unknown.result.details : null).toBeUndefined();
|
||||
expect(isImmediate(preparationFailure) ? text(preparationFailure.result) : "").toBe("cannot prepare");
|
||||
expect(isImmediate(invalid) ? text(invalid.result) : "").toContain('Validation failed for tool "echo"');
|
||||
});
|
||||
|
||||
it("blocks calls and revalidates replacement arguments", () => {
|
||||
const prepared = prepareToolCall(call(), [tool()]);
|
||||
if (isImmediate(prepared)) throw new Error("expected prepared call");
|
||||
|
||||
const blocked = applyBeforeToolDecision(prepared, {
|
||||
block: { reason: "denied", terminate: true },
|
||||
});
|
||||
const replaced = applyBeforeToolDecision(prepared, { args: { value: "replacement" } });
|
||||
const invalid = applyBeforeToolDecision(prepared, { args: {} });
|
||||
|
||||
expect(blocked).toMatchObject({ kind: "immediate", isError: true, terminate: true });
|
||||
expect(isImmediate(blocked) ? text(blocked.result) : "").toBe("denied");
|
||||
expect(isImmediate(replaced) ? undefined : replaced.args).toEqual({ value: "replacement" });
|
||||
expect(isImmediate(invalid)).toBe(true);
|
||||
});
|
||||
|
||||
it("executes with updates, passes the signal, and ignores late updates", async () => {
|
||||
const gate = effectGate();
|
||||
let lateUpdate: ((partial: AgentToolResult<{ value: string }>) => void) | undefined;
|
||||
const execute: AgentHarnessTool<undefined, typeof parameters, { value: string }>["execute"] = vi.fn(
|
||||
async (_id, args, onUpdate, _toolContext, _invocation, context) => {
|
||||
expect(context.abortSignal).toBe(gate.signal);
|
||||
onUpdate?.({ content: [{ type: "text", text: "partial" }], details: { value: args.value } });
|
||||
lateUpdate = onUpdate;
|
||||
return { content: [{ type: "text" as const, text: "done" }], details: { value: args.value } };
|
||||
},
|
||||
);
|
||||
const cleared = clearPrepared(prepareToolCall(call(), [tool({ execute })]));
|
||||
const updates: AgentToolResult<unknown>[] = [];
|
||||
|
||||
const result = await executeToolCall(
|
||||
cleared,
|
||||
gate,
|
||||
(update) => updates.push(update),
|
||||
undefined,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
lateUpdate?.({ content: [{ type: "text", text: "late" }], details: { value: "late" } });
|
||||
|
||||
expect(vi.mocked(execute)).toHaveBeenCalledOnce();
|
||||
expect(result.isError).toBe(false);
|
||||
expect(text(result.result)).toBe("done");
|
||||
expect(updates).toHaveLength(1);
|
||||
expect(text(updates[0])).toBe("partial");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"synchronous",
|
||||
() => {
|
||||
throw new Error("tool failed");
|
||||
},
|
||||
],
|
||||
[
|
||||
"asynchronous",
|
||||
async () => {
|
||||
throw new Error("tool failed");
|
||||
},
|
||||
],
|
||||
])("converts %s tool throws to error output", async (_kind, execute) => {
|
||||
const cleared = clearPrepared(prepareToolCall(call(), [tool({ execute })]));
|
||||
|
||||
const result = await executeToolCall(cleared, effectGate(), () => {}, undefined, invocation, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(result.isError).toBe(true);
|
||||
expect(text(result.result)).toBe("tool failed");
|
||||
});
|
||||
|
||||
it("lets abort-first gate refusal escape without invoking the tool", async () => {
|
||||
const execute = vi.fn(tool().execute);
|
||||
const cleared = clearPrepared(prepareToolCall(call(), [tool({ execute })]));
|
||||
const { gate, control } = createGate();
|
||||
control.beginAbort(Promise.resolve());
|
||||
|
||||
expect(() => executeToolCall(cleared, gate, () => {}, undefined, invocation, BACKGROUND_CONTEXT)).toThrow(
|
||||
AbortRequested,
|
||||
);
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("applies patches field by field and constructs the tool-result message", () => {
|
||||
const cleared = clearPrepared(prepareToolCall(call(), [tool()]));
|
||||
const originalUsage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 3,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
const replacementUsage = { ...originalUsage, input: 5, totalTokens: 7 };
|
||||
const executed: AgentToolResult<unknown> = {
|
||||
content: [{ type: "text", text: "original" }],
|
||||
details: { original: true },
|
||||
usage: originalUsage,
|
||||
addedToolNames: ["new-tool"],
|
||||
};
|
||||
|
||||
const finalized = finalizeToolCall(
|
||||
cleared,
|
||||
{ result: executed, isError: true },
|
||||
{
|
||||
content: [{ type: "text", text: "patched" }],
|
||||
details: { patched: true },
|
||||
usage: replacementUsage,
|
||||
isError: false,
|
||||
terminate: true,
|
||||
},
|
||||
);
|
||||
const before = Date.now();
|
||||
const message = createToolResultMessage(finalized);
|
||||
|
||||
expect(finalized).toMatchObject({ isError: false, terminate: true });
|
||||
expect(finalized.result).toMatchObject({
|
||||
content: [{ type: "text", text: "patched" }],
|
||||
details: { patched: true },
|
||||
usage: replacementUsage,
|
||||
addedToolNames: ["new-tool"],
|
||||
terminate: true,
|
||||
});
|
||||
expect(message).toMatchObject({
|
||||
role: "toolResult",
|
||||
toolCallId: "call-1",
|
||||
toolName: "echo",
|
||||
content: [{ type: "text", text: "patched" }],
|
||||
details: { patched: true },
|
||||
usage: replacementUsage,
|
||||
addedToolNames: ["new-tool"],
|
||||
isError: false,
|
||||
});
|
||||
expect(message.timestamp).toBeGreaterThanOrEqual(before);
|
||||
});
|
||||
|
||||
it("normalizes missing content from untyped tools", () => {
|
||||
const cleared = clearPrepared(prepareToolCall(call(), [tool()]));
|
||||
const finalized = finalizeToolCall(
|
||||
cleared,
|
||||
{ result: { content: undefined, details: {} } as never, isError: false },
|
||||
undefined,
|
||||
);
|
||||
|
||||
expect(createToolResultMessage(finalized).content).toEqual([]);
|
||||
});
|
||||
});
|
||||
110
third_party/pi/packages/agent/test/harness/gating-storage.test.ts
vendored
Normal file
110
third_party/pi/packages/agent/test/harness/gating-storage.test.ts
vendored
Normal file
@@ -0,0 +1,110 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT, type Context } from "../../src/harness/context.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import { CommitDiscarded, GatingStorage, InstrumentedStorage } from "../../src/harness/session/testing/index.ts";
|
||||
import type { CommitResult, Write } from "../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
import { deferred } from "./runtime/test-utils.ts";
|
||||
|
||||
class ControlledLandingStorage extends MemoryStorage {
|
||||
readonly started = deferred();
|
||||
readonly release = deferred();
|
||||
|
||||
override async commit(writes: Write[], context: Context): Promise<CommitResult> {
|
||||
this.started.resolve();
|
||||
await this.release.promise;
|
||||
return super.commit(writes, context);
|
||||
}
|
||||
}
|
||||
|
||||
describe("GatingStorage", () => {
|
||||
it("bypasses setup writes until armed and waits for commits parked after waitPending", async () => {
|
||||
const storage = new GatingStorage(new MemoryStorage({ now: () => 10 }));
|
||||
await storage.commit([storedValues.setValue(storedValues.sessionName, "setup")], BACKGROUND_CONTEXT);
|
||||
expect(storage.pending()).toBe(0);
|
||||
|
||||
storage.arm();
|
||||
let waitingResolved = false;
|
||||
const waiting = storage.waitPending().then(() => {
|
||||
waitingResolved = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(waitingResolved).toBe(false);
|
||||
const commit = storage.commit([storedValues.setValue(storedValues.sessionName, "parked")], BACKGROUND_CONTEXT);
|
||||
await waiting;
|
||||
expect(storage.pending()).toBe(1);
|
||||
expect((await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("setup");
|
||||
|
||||
await storage.next();
|
||||
await commit;
|
||||
expect(storage.pending()).toBe(0);
|
||||
expect((await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("parked");
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("releases in FIFO order and next resolves only after the backend write lands", async () => {
|
||||
const delegate = new ControlledLandingStorage({ now: () => 10 });
|
||||
const storage = new GatingStorage(delegate);
|
||||
storage.arm();
|
||||
const first = storage.commit([storedValues.setValue(storedValues.sessionName, "first")], BACKGROUND_CONTEXT);
|
||||
const second = storage.commit([storedValues.setValue(storedValues.sessionName, "second")], BACKGROUND_CONTEXT);
|
||||
await storage.waitPending(2);
|
||||
expect(storage.pending()).toBe(2);
|
||||
|
||||
let released = false;
|
||||
const next = storage.next().then(() => {
|
||||
released = true;
|
||||
});
|
||||
await delegate.started.promise;
|
||||
await Promise.resolve();
|
||||
expect(released).toBe(false);
|
||||
expect(storage.pending()).toBe(1);
|
||||
delegate.release.resolve();
|
||||
await next;
|
||||
await first;
|
||||
expect(released).toBe(true);
|
||||
expect((await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("first");
|
||||
|
||||
await storage.next();
|
||||
await second;
|
||||
expect((await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("second");
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("permanently rejects parked, waiting, and later commits after discard", async () => {
|
||||
const storage = new GatingStorage(new MemoryStorage());
|
||||
storage.arm();
|
||||
const waiting = storage.waitPending(2);
|
||||
const parked = storage.commit([storedValues.setValue(storedValues.sessionName, "lost")], BACKGROUND_CONTEXT);
|
||||
await storage.waitPending();
|
||||
expect(storage.pending()).toBe(1);
|
||||
|
||||
storage.discard();
|
||||
expect(storage.pending()).toBe(0);
|
||||
await expect(parked).rejects.toBeInstanceOf(CommitDiscarded);
|
||||
await expect(waiting).rejects.toBeInstanceOf(CommitDiscarded);
|
||||
await expect(storage.next()).rejects.toBeInstanceOf(CommitDiscarded);
|
||||
await expect(storage.commit([], BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(CommitDiscarded);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const unarmed = new GatingStorage(new MemoryStorage());
|
||||
unarmed.discard();
|
||||
await expect(unarmed.commit([], BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(CommitDiscarded);
|
||||
await unarmed.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("records attempts before gating parks them", async () => {
|
||||
const gating = new GatingStorage(new MemoryStorage());
|
||||
const storage = new InstrumentedStorage(gating);
|
||||
gating.arm();
|
||||
const writes = [storedValues.setValue(storedValues.sessionName, "recorded")];
|
||||
|
||||
const commit = storage.commit(writes, BACKGROUND_CONTEXT);
|
||||
expect(storage.getCommitAttempts()).toEqual([writes]);
|
||||
await gating.waitPending();
|
||||
expect(gating.pending()).toBe(1);
|
||||
await gating.next();
|
||||
await commit;
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
190
third_party/pi/packages/agent/test/harness/instrumented-storage.test.ts
vendored
Normal file
190
third_party/pi/packages/agent/test/harness/instrumented-storage.test.ts
vendored
Normal file
@@ -0,0 +1,190 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import * as sessionWrites from "../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import { InstrumentedStorage } from "../../src/harness/session/testing/index.ts";
|
||||
import type { CommitResult, SessionStats, Write } from "../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
|
||||
const EMPTY_STATS: SessionStats = {
|
||||
messageCount: 0,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
};
|
||||
|
||||
class ControlledCommitStorage extends MemoryStorage {
|
||||
private readonly pending: Array<{
|
||||
resolve: (result: CommitResult) => void;
|
||||
reject: (error: unknown) => void;
|
||||
}> = [];
|
||||
private latestCommit: Promise<CommitResult> | undefined;
|
||||
|
||||
override commit(_transaction: Write[]): Promise<CommitResult> {
|
||||
this.latestCommit = new Promise((resolve, reject) => {
|
||||
this.pending.push({ resolve, reject });
|
||||
});
|
||||
return this.latestCommit;
|
||||
}
|
||||
|
||||
get admissionCount(): number {
|
||||
return this.pending.length;
|
||||
}
|
||||
|
||||
get lastCommit(): Promise<CommitResult> | undefined {
|
||||
return this.latestCommit;
|
||||
}
|
||||
|
||||
resolveNextCommit(result: CommitResult): void {
|
||||
const pending = this.pending.shift();
|
||||
if (pending === undefined) throw new Error("No pending commit");
|
||||
pending.resolve(result);
|
||||
}
|
||||
|
||||
rejectNextCommit(error: unknown): void {
|
||||
const pending = this.pending.shift();
|
||||
if (pending === undefined) throw new Error("No pending commit");
|
||||
pending.reject(error);
|
||||
}
|
||||
}
|
||||
|
||||
describe("InstrumentedStorage", () => {
|
||||
it("records commit attempts synchronously in admission order before settlement", async () => {
|
||||
const delegate = new ControlledCommitStorage();
|
||||
const storage = new InstrumentedStorage(delegate);
|
||||
const firstTransaction: Write[] = [storedValues.setValue(storedValues.sessionName, "first")];
|
||||
const secondTransaction: Write[] = [storedValues.setValue(storedValues.sessionName, "second")];
|
||||
|
||||
const firstCommit = storage.commit(firstTransaction, BACKGROUND_CONTEXT);
|
||||
expect(firstCommit).toBe(delegate.lastCommit);
|
||||
expect(storage.getCommitAttempts()).toEqual([firstTransaction]);
|
||||
const secondCommit = storage.commit(secondTransaction, BACKGROUND_CONTEXT);
|
||||
expect(secondCommit).toBe(delegate.lastCommit);
|
||||
expect(storage.getCommitAttempts()).toEqual([firstTransaction, secondTransaction]);
|
||||
|
||||
const firstResult = { firstSeq: 1, seqs: [1], timestamp: 10, stats: EMPTY_STATS };
|
||||
delegate.resolveNextCommit(firstResult);
|
||||
expect(await firstCommit).toBe(firstResult);
|
||||
expect(storage.getCommitAttempts()).toEqual([firstTransaction, secondTransaction]);
|
||||
const secondResult = { firstSeq: 2, seqs: [2], timestamp: 20, stats: EMPTY_STATS };
|
||||
delegate.resolveNextCommit(secondResult);
|
||||
expect(await secondCommit).toBe(secondResult);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
it("records the transaction reference passed to the delegate", async () => {
|
||||
const delegate = new ControlledCommitStorage();
|
||||
const storage = new InstrumentedStorage(delegate);
|
||||
const transaction: Write[] = [storedValues.setValue(storedValues.sessionName, "value")];
|
||||
|
||||
const commit = storage.commit(transaction, BACKGROUND_CONTEXT);
|
||||
expect(storage.getCommitAttempts()[0]).toBe(transaction);
|
||||
delegate.resolveNextCommit({ firstSeq: 1, seqs: [1], timestamp: 10, stats: EMPTY_STATS });
|
||||
await commit;
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("clears recorded attempts between phases without affecting the delegate", async () => {
|
||||
const delegate = new MemoryStorage({ now: () => 100 });
|
||||
const storage = new InstrumentedStorage(delegate);
|
||||
await storage.commit([storedValues.setValue(storedValues.sessionName, "first")], BACKGROUND_CONTEXT);
|
||||
|
||||
storage.clearCommitAttempts();
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
expect(await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toMatchObject({ value: "first" });
|
||||
|
||||
const secondTransaction: Write[] = [storedValues.setValue(storedValues.sessionName, "second")];
|
||||
await storage.commit(secondTransaction, BACKGROUND_CONTEXT);
|
||||
expect(storage.getCommitAttempts()).toEqual([secondTransaction]);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("delegates every read and query without recording synthetic writes", async () => {
|
||||
const delegate = new MemoryStorage({ now: () => 100 });
|
||||
const storage = new InstrumentedStorage(delegate);
|
||||
const events = storedValues.list<string>("test.events");
|
||||
await storage.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({ id: "root", parentId: null, type: "custom", customType: "note" }),
|
||||
storedValues.setValue(storedValues.sessionName, "session"),
|
||||
storedValues.appendList(events, "event"),
|
||||
sessionWrites.insertUsage({
|
||||
id: "usage",
|
||||
adjustment: false,
|
||||
usage: {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 3,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(await storage.getEntries(["root"], BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.getEntries(["root"], BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.getValue(storedValues.sessionName, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.scanValues(storedValues.sessionName, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.scanValues(storedValues.sessionName, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.readList(events, undefined, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.readList(events, undefined, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.scanBranch({ start: "root" }, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.scanBranch({ start: "root" }, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.scanBranchStructure({ start: "root" }, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.scanBranchStructure({ start: "root" }, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.scanEntries({ order: "asc" }, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.scanEntries({ order: "asc" }, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.scanUsage({ order: "asc" }, BACKGROUND_CONTEXT)).toEqual(
|
||||
await delegate.scanUsage({ order: "asc" }, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(await storage.getStats(BACKGROUND_CONTEXT)).toEqual(await delegate.getStats(BACKGROUND_CONTEXT));
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("records list appends without reading the target list", async () => {
|
||||
const delegate = new MemoryStorage({ now: () => 100 });
|
||||
const readList = vi.spyOn(delegate, "readList");
|
||||
const storage = new InstrumentedStorage(delegate);
|
||||
const events = storedValues.list<string>("test.events");
|
||||
|
||||
await storage.commit([storedValues.appendList(events, "event")], BACKGROUND_CONTEXT);
|
||||
|
||||
expect(readList).not.toHaveBeenCalled();
|
||||
expect(storage.getCommitAttempts()).toEqual([
|
||||
[{ kind: "list", op: "append", namespace: "test.events", key: "", value: "event" }],
|
||||
]);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("delegates close idempotence and admitted commit draining", async () => {
|
||||
const delegate = new MemoryStorage({ now: () => 100 });
|
||||
const storage = new InstrumentedStorage(delegate);
|
||||
const admitted = storage.commit(
|
||||
[storedValues.setValue(storedValues.sessionName, "admitted")],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const firstClose = storage.close(BACKGROUND_CONTEXT);
|
||||
const secondClose = storage.close(BACKGROUND_CONTEXT);
|
||||
await admitted;
|
||||
await Promise.all([firstClose, secondClose]);
|
||||
await expect(storage.getStats(BACKGROUND_CONTEXT)).rejects.toThrow("MemoryStorage is closed");
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
61
third_party/pi/packages/agent/test/harness/jsonl-session-repo-conformance.test.ts
vendored
Normal file
61
third_party/pi/packages/agent/test/harness/jsonl-session-repo-conformance.test.ts
vendored
Normal file
@@ -0,0 +1,61 @@
|
||||
import { describe, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import { type JsonlSessionMetadata, JsonlSessionRepo } from "../../src/harness/session/jsonl/index.ts";
|
||||
import {
|
||||
type ConformanceCase,
|
||||
createSessionRepoForkBehaviorConformance,
|
||||
createSessionRepoForkSourceSnapshotConformance,
|
||||
createSessionRepoLifecycleConformance,
|
||||
createSessionRepoMessageConformance,
|
||||
createSessionRepoOwnershipConformance,
|
||||
} from "../../src/harness/session/testing/index.ts";
|
||||
import type { ForkOptions } from "../../src/harness/session/types.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
const CONFORMANCE_CWD = "/workspace";
|
||||
|
||||
function registerConformance(name: string, cases: readonly ConformanceCase[]): void {
|
||||
describe(name, () => {
|
||||
for (const group of new Set(cases.map((testCase) => testCase.group))) {
|
||||
describe(group, () => {
|
||||
for (const testCase of cases.filter((candidate) => candidate.group === group)) {
|
||||
it(testCase.name, () => testCase.run());
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let jsonlRepo: JsonlSessionRepo;
|
||||
async function createConformanceRepo() {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
jsonlRepo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
return {
|
||||
create: (options: { id?: string; parentSessionId?: string }) =>
|
||||
jsonlRepo.create({ ...options, cwd: CONFORMANCE_CWD }, BACKGROUND_CONTEXT),
|
||||
open: (metadata: JsonlSessionMetadata) => jsonlRepo.open(metadata, BACKGROUND_CONTEXT),
|
||||
list: () => jsonlRepo.list({ cwd: CONFORMANCE_CWD }, BACKGROUND_CONTEXT),
|
||||
delete: (metadata: JsonlSessionMetadata) => jsonlRepo.delete(metadata, BACKGROUND_CONTEXT),
|
||||
fork: (source: JsonlSessionMetadata, options: ForkOptions) => jsonlRepo.fork(source, options, BACKGROUND_CONTEXT),
|
||||
};
|
||||
}
|
||||
|
||||
registerConformance("JsonlSessionRepo conformance", [
|
||||
...createSessionRepoLifecycleConformance<JsonlSessionMetadata>(createConformanceRepo, () =>
|
||||
jsonlRepo.close(BACKGROUND_CONTEXT),
|
||||
),
|
||||
...createSessionRepoOwnershipConformance<JsonlSessionMetadata>(createConformanceRepo, () =>
|
||||
jsonlRepo.close(BACKGROUND_CONTEXT),
|
||||
),
|
||||
...createSessionRepoMessageConformance<JsonlSessionMetadata>(createConformanceRepo, () =>
|
||||
jsonlRepo.close(BACKGROUND_CONTEXT),
|
||||
),
|
||||
...createSessionRepoForkBehaviorConformance<JsonlSessionMetadata>(createConformanceRepo, () =>
|
||||
jsonlRepo.close(BACKGROUND_CONTEXT),
|
||||
),
|
||||
...createSessionRepoForkSourceSnapshotConformance<JsonlSessionMetadata>(createConformanceRepo, () =>
|
||||
jsonlRepo.close(BACKGROUND_CONTEXT),
|
||||
),
|
||||
]);
|
||||
199
third_party/pi/packages/agent/test/harness/jsonl-session-repo.test.ts
vendored
Normal file
199
third_party/pi/packages/agent/test/harness/jsonl-session-repo.test.ts
vendored
Normal file
@@ -0,0 +1,199 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT, type Context } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import { JSONL_STORAGE_VERSION, JsonlSessionRepo } from "../../src/harness/session/jsonl/index.ts";
|
||||
import { sessionName, setValue } from "../../src/harness/session/values.ts";
|
||||
import { getOrThrow } from "../../src/harness/types.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
class AtomicPublicationNodeExecutionEnv extends NodeExecutionEnv {
|
||||
publication:
|
||||
| { sourcePath: string; destinationPath: string; destinationExisted: boolean; stagedContent: string }
|
||||
| undefined;
|
||||
|
||||
override async renameFile(sourcePath: string, destinationPath: string, context: Context) {
|
||||
const destinationExists = await super.exists(destinationPath, context);
|
||||
const stagedContent = await super.readTextFile(sourcePath, context);
|
||||
if (destinationExists.ok && stagedContent.ok) {
|
||||
this.publication = {
|
||||
sourcePath,
|
||||
destinationPath,
|
||||
destinationExisted: destinationExists.value,
|
||||
stagedContent: stagedContent.value,
|
||||
};
|
||||
}
|
||||
return super.renameFile(sourcePath, destinationPath, context);
|
||||
}
|
||||
}
|
||||
|
||||
describe("JsonlSessionRepo cwd-scoped lifecycle", () => {
|
||||
it("persists metadata and filters discovery by cwd", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
const session = await repo.create(
|
||||
{ id: "child", cwd: "/workspace", parentSessionId: "parent" },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const metadata = session.metadata;
|
||||
|
||||
expect(metadata).toMatchObject({
|
||||
id: "child",
|
||||
createdAt: NOW,
|
||||
storageVersion: JSONL_STORAGE_VERSION,
|
||||
cwd: "/workspace",
|
||||
parentSessionId: "parent",
|
||||
});
|
||||
expect(metadata.path).toContain("/sessions/--workspace--/");
|
||||
expect(metadata.path.endsWith("_child.jsonl")).toBe(true);
|
||||
expect(Number.isFinite(metadata.modifiedAt)).toBe(true);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await repo.list({ cwd: "/other" }, BACKGROUND_CONTEXT)).toEqual([]);
|
||||
expect(await repo.list({ cwd: "/workspace" }, BACKGROUND_CONTEXT)).toEqual([metadata]);
|
||||
const firstLine = getOrThrow(
|
||||
await fileSystem.readTextLines(metadata.path, { maxLines: 1 }, BACKGROUND_CONTEXT),
|
||||
)[0];
|
||||
expect(JSON.parse(firstLine!)).toEqual({
|
||||
v: 4,
|
||||
kind: "header",
|
||||
id: "child",
|
||||
storageVersion: JSONL_STORAGE_VERSION,
|
||||
createdAt: NOW,
|
||||
cwd: "/workspace",
|
||||
parentSessionId: "parent",
|
||||
});
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("atomically publishes a branchless session header", async () => {
|
||||
const fileSystem = new AtomicPublicationNodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
const session = await repo.create({ id: "session", cwd: "/workspace" }, BACKGROUND_CONTEXT);
|
||||
|
||||
const publication = fileSystem.publication;
|
||||
if (publication === undefined) throw new Error("Expected atomic session publication");
|
||||
expect(publication.destinationPath).toBe(session.metadata.path);
|
||||
expect(publication.destinationExisted).toBe(false);
|
||||
const lines = publication.stagedContent.trimEnd().split("\n");
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(JSON.parse(lines[0]!)).toMatchObject({ kind: "header", id: "session" });
|
||||
expect(getOrThrow(await fileSystem.readTextFile(session.metadata.path, BACKGROUND_CONTEXT))).toBe(
|
||||
publication.stagedContent,
|
||||
);
|
||||
expect(getOrThrow(await fileSystem.exists(publication.sourcePath, BACKGROUND_CONTEXT))).toBe(false);
|
||||
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("keeps an explicit Session mutation through commit until end", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
const session = await repo.create({ id: "session", cwd: "/workspace" }, BACKGROUND_CONTEXT);
|
||||
const mutation = await session.beginMutation(BACKGROUND_CONTEXT);
|
||||
let queuedStarted = false;
|
||||
const queued = session.mutate(() => {
|
||||
queuedStarted = true;
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
const result = await mutation.commit([setValue(sessionName, "explicit")], BACKGROUND_CONTEXT);
|
||||
expect(result.seqs).toHaveLength(1);
|
||||
expect(queuedStarted).toBe(false);
|
||||
expect(await mutation.getValue(sessionName, BACKGROUND_CONTEXT)).toMatchObject({ value: "explicit" });
|
||||
await mutation.end(BACKGROUND_CONTEXT);
|
||||
await queued;
|
||||
expect(queuedStarted).toBe(true);
|
||||
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
const reopened = await repo.open(session.metadata, BACKGROUND_CONTEXT);
|
||||
expect(await reopened.getName(BACKGROUND_CONTEXT)).toBe("explicit");
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects unsupported storage versions without repairing a torn tail", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
const session = await repo.create({ id: "future", cwd: "/workspace" }, BACKGROUND_CONTEXT);
|
||||
const metadata = session.metadata;
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const lines = getOrThrow(await fileSystem.readTextFile(metadata.path, BACKGROUND_CONTEXT))
|
||||
.trimEnd()
|
||||
.split("\n");
|
||||
const unsupportedVersion = JSONL_STORAGE_VERSION + 1;
|
||||
lines[0] = JSON.stringify({ ...JSON.parse(lines[0]!), storageVersion: unsupportedVersion });
|
||||
const unsupportedContent = `${lines.join("\n")}\n{"kind":"entry"`;
|
||||
getOrThrow(await fileSystem.writeFile(metadata.path, unsupportedContent, BACKGROUND_CONTEXT));
|
||||
|
||||
await expect(repo.open(metadata, BACKGROUND_CONTEXT)).rejects.toThrow(
|
||||
`unsupported storage version ${unsupportedVersion}`,
|
||||
);
|
||||
expect(getOrThrow(await fileSystem.readTextFile(metadata.path, BACKGROUND_CONTEXT))).toBe(unsupportedContent);
|
||||
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("keeps fork destinations claimed until close and rejects deleting open sessions", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
const source = await repo.create({ id: "source", cwd: "/workspace" }, BACKGROUND_CONTEXT);
|
||||
const fork = await repo.fork(source.metadata, { id: "fork", scope: "tree" }, BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(repo.open(fork.metadata, BACKGROUND_CONTEXT)).rejects.toThrow("already open");
|
||||
await expect(repo.delete(fork.metadata, BACKGROUND_CONTEXT)).rejects.toThrow("open");
|
||||
await fork.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const reopened = await repo.open(fork.metadata, BACKGROUND_CONTEXT);
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
await repo.delete(fork.metadata, BACKGROUND_CONTEXT);
|
||||
await expect(repo.open(fork.metadata, BACKGROUND_CONTEXT)).rejects.toThrow("does not exist");
|
||||
|
||||
await source.close(BACKGROUND_CONTEXT);
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects concurrent creates for the same working-directory id", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
|
||||
const results = await Promise.allSettled([
|
||||
repo.create({ id: "session", cwd: "/workspace" }, BACKGROUND_CONTEXT),
|
||||
repo.create({ id: "session", cwd: "/workspace" }, BACKGROUND_CONTEXT),
|
||||
]);
|
||||
|
||||
expect(results.filter((result) => result.status === "fulfilled")).toHaveLength(1);
|
||||
expect(results.filter((result) => result.status === "rejected")).toHaveLength(1);
|
||||
expect(await repo.list({ cwd: "/workspace" }, BACKGROUND_CONTEXT)).toHaveLength(1);
|
||||
for (const result of results) {
|
||||
if (result.status === "fulfilled") await result.value.close(BACKGROUND_CONTEXT);
|
||||
}
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("allows the same id to be active in different working directories", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const repo = new JsonlSessionRepo({ fileSystem, sessionsRoot: "sessions", now: () => NOW });
|
||||
const first = await repo.create({ id: "shared", cwd: "/workspace-a" }, BACKGROUND_CONTEXT);
|
||||
const second = await repo.create({ id: "shared", cwd: "/workspace-b" }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(first.metadata.path).not.toBe(second.metadata.path);
|
||||
await expect(repo.create({ id: "shared", cwd: "/workspace-a" }, BACKGROUND_CONTEXT)).rejects.toThrow(
|
||||
"already exists",
|
||||
);
|
||||
expect((await repo.list(undefined, BACKGROUND_CONTEXT)).map(({ cwd, id }) => ({ cwd, id }))).toEqual([
|
||||
{ cwd: "/workspace-a", id: "shared" },
|
||||
{ cwd: "/workspace-b", id: "shared" },
|
||||
]);
|
||||
|
||||
await Promise.all([first.close(BACKGROUND_CONTEXT), second.close(BACKGROUND_CONTEXT)]);
|
||||
const reopened = await Promise.all([
|
||||
repo.open(first.metadata, BACKGROUND_CONTEXT),
|
||||
repo.open(second.metadata, BACKGROUND_CONTEXT),
|
||||
]);
|
||||
await Promise.all(reopened.map((session) => session.close(BACKGROUND_CONTEXT)));
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
48
third_party/pi/packages/agent/test/harness/jsonl-storage-conformance.test.ts
vendored
Normal file
48
third_party/pi/packages/agent/test/harness/jsonl-storage-conformance.test.ts
vendored
Normal file
@@ -0,0 +1,48 @@
|
||||
import { describe, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import { JSONL_FORMAT_VERSION, JsonlStorage } from "../../src/harness/session/jsonl/index.ts";
|
||||
import {
|
||||
type ConformanceCase,
|
||||
createStorageConformance,
|
||||
type StorageFixture,
|
||||
} from "../../src/harness/session/testing/index.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
function registerConformance(name: string, cases: readonly ConformanceCase[]): void {
|
||||
describe(name, () => {
|
||||
for (const group of new Set(cases.map((testCase) => testCase.group))) {
|
||||
describe(group, () => {
|
||||
for (const testCase of cases.filter((candidate) => candidate.group === group)) {
|
||||
it(testCase.name, () => testCase.run());
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
registerConformance(
|
||||
"JsonlStorage conformance",
|
||||
createStorageConformance(async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const storage = await JsonlStorage.create(
|
||||
{ fileSystem, path: "session.jsonl", now: () => NOW },
|
||||
{
|
||||
v: JSONL_FORMAT_VERSION,
|
||||
kind: "header",
|
||||
id: "session",
|
||||
storageVersion: 1,
|
||||
createdAt: NOW,
|
||||
cwd: "/workspace",
|
||||
},
|
||||
[],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return {
|
||||
storage,
|
||||
[Symbol.asyncDispose]: () => storage.close(BACKGROUND_CONTEXT),
|
||||
} satisfies StorageFixture;
|
||||
}),
|
||||
);
|
||||
387
third_party/pi/packages/agent/test/harness/jsonl-storage.test.ts
vendored
Normal file
387
third_party/pi/packages/agent/test/harness/jsonl-storage.test.ts
vendored
Normal file
@@ -0,0 +1,387 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import * as sessionWrites from "../../src/harness/session/commit.ts";
|
||||
import type { ForkDestinationSnapshot } from "../../src/harness/session/fork.ts";
|
||||
import { JSONL_FORMAT_VERSION, JsonlStorage, type JsonlStorageHeader } from "../../src/harness/session/jsonl/index.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
import { getOrThrow } from "../../src/harness/types.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
function header(id: string): JsonlStorageHeader {
|
||||
return {
|
||||
v: JSONL_FORMAT_VERSION,
|
||||
kind: "header",
|
||||
id,
|
||||
storageVersion: 1,
|
||||
createdAt: NOW,
|
||||
cwd: "/workspace",
|
||||
};
|
||||
}
|
||||
|
||||
function stored<T>(address: storedValues.Value<T>, value: T, seq: number): storedValues.StoredValue<unknown> {
|
||||
return {
|
||||
address: storedValues.value<unknown>(address.namespace, address.key),
|
||||
value,
|
||||
seq,
|
||||
};
|
||||
}
|
||||
|
||||
function preparedSnapshot(): ForkDestinationSnapshot {
|
||||
return {
|
||||
entries: new Map([
|
||||
[
|
||||
"root",
|
||||
{
|
||||
id: "root",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "hello", timestamp: 1 },
|
||||
seq: 1,
|
||||
timestamp: NOW,
|
||||
},
|
||||
],
|
||||
]),
|
||||
scalarValues: [
|
||||
stored(storedValues.branchTip("main"), "root", 2),
|
||||
stored(storedValues.laneState("main"), { currentOperationId: null, lastOperationId: null, inbox: [] }, 3),
|
||||
stored(storedValues.sessionName, "forked", 4),
|
||||
],
|
||||
nextSeq: 9,
|
||||
};
|
||||
}
|
||||
|
||||
describe("JsonlStorage snapshot creation", () => {
|
||||
it("atomically publishes, advances, and reopens a prepared snapshot", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const options = { fileSystem, path: "fork.jsonl", now: () => NOW };
|
||||
const storage = await JsonlStorage.createFromForkSnapshot(
|
||||
options,
|
||||
header("fork"),
|
||||
preparedSnapshot(),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const lines = getOrThrow(await fileSystem.readTextFile("fork.jsonl", BACKGROUND_CONTEXT))
|
||||
.trimEnd()
|
||||
.split("\n");
|
||||
expect(lines.slice(1).every((line) => !Array.isArray(JSON.parse(line)))).toBe(true);
|
||||
expect((await storage.getEntries(["root"], BACKGROUND_CONTEXT)).get("root")?.timestamp).toBe(NOW);
|
||||
expect((await storage.getValue(storedValues.branchTip("main"), BACKGROUND_CONTEXT))?.value).toBe("root");
|
||||
expect((await storage.getValue(storedValues.laneState("main"), BACKGROUND_CONTEXT))?.value).toEqual({
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
});
|
||||
expect((await storage.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("forked");
|
||||
expect(await storage.getStats(BACKGROUND_CONTEXT)).toEqual({
|
||||
messageCount: 1,
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
});
|
||||
expect(await storage.scanUsage({ order: "asc" }, BACKGROUND_CONTEXT)).toEqual([]);
|
||||
expect(
|
||||
(await storage.commit([storedValues.setValue(storedValues.sessionName, "updated")], BACKGROUND_CONTEXT))
|
||||
.firstSeq,
|
||||
).toBe(9);
|
||||
expect(getOrThrow(await fileSystem.exists("fork.jsonl.tmp", BACKGROUND_CONTEXT))).toBe(false);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const reopened = await JsonlStorage.open(options, BACKGROUND_CONTEXT);
|
||||
expect((await reopened.getEntries(["root"], BACKGROUND_CONTEXT)).has("root")).toBe(true);
|
||||
expect((await reopened.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("updated");
|
||||
expect((await reopened.commit([], BACKGROUND_CONTEXT)).firstSeq).toBe(10);
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("removes the temporary file when publication fails", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
getOrThrow(await fileSystem.createDir("blocked.jsonl", undefined, BACKGROUND_CONTEXT));
|
||||
|
||||
await expect(
|
||||
JsonlStorage.createFromForkSnapshot(
|
||||
{ fileSystem, path: "blocked.jsonl", now: () => NOW },
|
||||
header("blocked"),
|
||||
preparedSnapshot(),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow("Failed to publish JSONL storage");
|
||||
expect(getOrThrow(await fileSystem.exists("blocked.jsonl.tmp", BACKGROUND_CONTEXT))).toBe(false);
|
||||
expect(getOrThrow(await fileSystem.fileInfo("blocked.jsonl", BACKGROUND_CONTEXT))).toMatchObject({
|
||||
kind: "directory",
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("JsonlStorage persistence", () => {
|
||||
it("replays whole-list deletion without resurrecting earlier appends", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const options = { fileSystem, path: "list-delete.jsonl", now: () => NOW };
|
||||
const events = storedValues.list<string>("test.events");
|
||||
const storage = await JsonlStorage.create(options, header("list-delete"), [], BACKGROUND_CONTEXT);
|
||||
await storage.commit(
|
||||
[storedValues.appendList(events, "first"), storedValues.appendList(events, "second")],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await storage.commit([storedValues.deleteList(events)], BACKGROUND_CONTEXT);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const reopened = await JsonlStorage.open(options, BACKGROUND_CONTEXT);
|
||||
expect(await reopened.readList(events, undefined, BACKGROUND_CONTEXT)).toEqual([]);
|
||||
const recreated = await reopened.commit([storedValues.appendList(events, "after")], BACKGROUND_CONTEXT);
|
||||
expect(recreated.firstSeq).toBe(4);
|
||||
expect(await reopened.readList(events, undefined, BACKGROUND_CONTEXT)).toEqual([{ seq: 4, value: "after" }]);
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("writes one line per transaction and replays stamped state", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const options = { fileSystem, path: "session.jsonl", now: () => NOW };
|
||||
const storage = await JsonlStorage.create(options, header("round-trip"), [], BACKGROUND_CONTEXT);
|
||||
const committed = await storage.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({
|
||||
id: "root",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "hello", timestamp: 1 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "root"),
|
||||
sessionWrites.insertUsage({
|
||||
id: "usage",
|
||||
entryId: "root",
|
||||
adjustment: false,
|
||||
usage: {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 3,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await storage.commit([storedValues.setValue(storedValues.sessionName, "name")], BACKGROUND_CONTEXT);
|
||||
|
||||
const lines = getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))
|
||||
.trimEnd()
|
||||
.split("\n");
|
||||
expect(JSON.parse(lines[0]!)).toEqual(header("round-trip"));
|
||||
expect(JSON.parse(lines[1]!)).toHaveLength(3);
|
||||
expect(Array.isArray(JSON.parse(lines[2]!))).toBe(false);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
|
||||
const reopened = await JsonlStorage.open(options, BACKGROUND_CONTEXT);
|
||||
expect((await reopened.getEntries(["root"], BACKGROUND_CONTEXT)).get("root")).toEqual({
|
||||
id: "root",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "hello", timestamp: 1 },
|
||||
seq: committed.seqs[0],
|
||||
timestamp: committed.timestamp,
|
||||
});
|
||||
expect(await reopened.getValue(storedValues.branchTip("main"), BACKGROUND_CONTEXT)).toEqual({
|
||||
address: storedValues.branchTip("main"),
|
||||
value: "root",
|
||||
seq: committed.seqs[1],
|
||||
});
|
||||
expect(
|
||||
(await reopened.scanUsage({ order: "asc" }, BACKGROUND_CONTEXT)).map(({ id, seq }) => ({ id, seq })),
|
||||
).toEqual([{ id: "usage", seq: committed.seqs[2] }]);
|
||||
const historicalStats = {
|
||||
messageCount: 1,
|
||||
usage: {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 3,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
};
|
||||
expect(await reopened.getStats(BACKGROUND_CONTEXT)).toEqual(historicalStats);
|
||||
const next = await reopened.commit([], BACKGROUND_CONTEXT);
|
||||
expect(next.firstSeq).toBe(5);
|
||||
expect(next.stats).toEqual(historicalStats);
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("JsonlStorage snapshots", () => {
|
||||
it("captures one serialized boundary between commits", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const options = { fileSystem, path: "session.jsonl", now: () => NOW };
|
||||
const storage = await JsonlStorage.create(options, header("snapshot"), [], BACKGROUND_CONTEXT);
|
||||
|
||||
const firstCommit = storage.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({ id: "root", parentId: null, type: "custom", customType: "root" }),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "root"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const snapshot = storage.captureForkSource(BACKGROUND_CONTEXT);
|
||||
const secondCommit = storage.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({ id: "child", parentId: "root", type: "custom", customType: "child" }),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "child"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
await firstCommit;
|
||||
const captured = await snapshot;
|
||||
await secondCommit;
|
||||
|
||||
expect(captured.entries.map(({ id }) => id)).toEqual(["root"]);
|
||||
expect(
|
||||
captured.scalarValues.find(
|
||||
({ address }) => address.namespace === storedValues.branchTip("main").namespace && address.key === "main",
|
||||
)?.value,
|
||||
).toBe("root");
|
||||
expect((await storage.getEntries(["child"], BACKGROUND_CONTEXT)).has("child")).toBe(true);
|
||||
expect((await storage.getValue(storedValues.branchTip("main"), BACKGROUND_CONTEXT))?.value).toBe("child");
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("JsonlStorage torn tail", () => {
|
||||
function entryWrite(id: string) {
|
||||
return sessionWrites.insertEntry({
|
||||
id,
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: id, timestamp: 1 },
|
||||
});
|
||||
}
|
||||
|
||||
async function seed() {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const options = { fileSystem, path: "session.jsonl" as const, now: () => NOW };
|
||||
const storage = await JsonlStorage.create(options, header("torn"), [], BACKGROUND_CONTEXT);
|
||||
await storage.commit([entryWrite("kept")], BACKGROUND_CONTEXT);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
const prefix = getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT));
|
||||
return { fileSystem, options, prefix };
|
||||
}
|
||||
|
||||
it("discards an unterminated final object line and truncates before admitting writes", async () => {
|
||||
const { fileSystem, options, prefix } = await seed();
|
||||
await fileSystem.appendFile(
|
||||
"session.jsonl",
|
||||
JSON.stringify({
|
||||
kind: "entry",
|
||||
id: "torn",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "torn", timestamp: 1 },
|
||||
seq: 2,
|
||||
timestamp: NOW,
|
||||
}),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const reopened = await JsonlStorage.open(options, BACKGROUND_CONTEXT);
|
||||
expect((await reopened.getEntries(["kept", "torn"], BACKGROUND_CONTEXT)).has("torn")).toBe(false);
|
||||
expect((await reopened.getEntries(["kept"], BACKGROUND_CONTEXT)).get("kept")?.id).toBe("kept");
|
||||
expect(getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))).toBe(prefix);
|
||||
expect(getOrThrow(await fileSystem.exists("session.jsonl.tmp", BACKGROUND_CONTEXT))).toBe(false);
|
||||
|
||||
const next = await reopened.commit([entryWrite("after")], BACKGROUND_CONTEXT);
|
||||
expect(next.firstSeq).toBe(2);
|
||||
expect((await reopened.getEntries(["after"], BACKGROUND_CONTEXT)).get("after")?.seq).toBe(2);
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("discards a torn array line wholly, including list elements", async () => {
|
||||
const { fileSystem, options, prefix } = await seed();
|
||||
const events = storedValues.list<string>("test.events");
|
||||
await fileSystem.appendFile(
|
||||
"session.jsonl",
|
||||
JSON.stringify([
|
||||
{
|
||||
kind: "entry",
|
||||
id: "torn-a",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "torn-a", timestamp: 1 },
|
||||
seq: 2,
|
||||
timestamp: NOW,
|
||||
},
|
||||
storedValues.setValue(storedValues.sessionName, "lost"),
|
||||
storedValues.appendList(events, "lost"),
|
||||
]),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const reopened = await JsonlStorage.open(options, BACKGROUND_CONTEXT);
|
||||
expect((await reopened.getEntries(["torn-a"], BACKGROUND_CONTEXT)).has("torn-a")).toBe(false);
|
||||
expect(await reopened.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(await reopened.readList(events, undefined, BACKGROUND_CONTEXT)).toEqual([]);
|
||||
expect(getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))).toBe(prefix);
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects a malformed interior line without rewriting", async () => {
|
||||
const { fileSystem, options, prefix } = await seed();
|
||||
const corrupted = `${prefix}not-json\n${JSON.stringify(storedValues.setValue(storedValues.sessionName, "after"))}\n`;
|
||||
await fileSystem.writeFile("session.jsonl", corrupted, BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(JsonlStorage.open(options, BACKGROUND_CONTEXT)).rejects.toThrow(/line 3/);
|
||||
expect(getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))).toBe(corrupted);
|
||||
expect(getOrThrow(await fileSystem.exists("session.jsonl.tmp", BACKGROUND_CONTEXT))).toBe(false);
|
||||
});
|
||||
|
||||
it("rejects the unsupported pre-WP01 scalar record spelling", async () => {
|
||||
const { fileSystem, options, prefix } = await seed();
|
||||
const legacyKind = ["reg", "ister"].join("");
|
||||
const corrupted = `${prefix}${JSON.stringify({
|
||||
kind: legacyKind,
|
||||
op: "set",
|
||||
seq: 2,
|
||||
namespace: "legacy.value",
|
||||
key: "state",
|
||||
value: true,
|
||||
})}\n`;
|
||||
await fileSystem.writeFile("session.jsonl", corrupted, BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(JsonlStorage.open(options, BACKGROUND_CONTEXT)).rejects.toThrow(/line 3/);
|
||||
expect(getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))).toBe(corrupted);
|
||||
});
|
||||
|
||||
it("rejects a complete malformed final line without rewriting", async () => {
|
||||
const { fileSystem, options, prefix } = await seed();
|
||||
const corrupted = `${prefix}not-json\n`;
|
||||
await fileSystem.writeFile("session.jsonl", corrupted, BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(JsonlStorage.open(options, BACKGROUND_CONTEXT)).rejects.toThrow(/line 3/);
|
||||
expect(getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))).toBe(corrupted);
|
||||
});
|
||||
|
||||
it("rejects a complete final line with invalid transaction framing", async () => {
|
||||
const { fileSystem, options, prefix } = await seed();
|
||||
const corrupted = `${prefix}${JSON.stringify({ kind: "nope", seq: 2 })}\n`;
|
||||
await fileSystem.writeFile("session.jsonl", corrupted, BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(JsonlStorage.open(options, BACKGROUND_CONTEXT)).rejects.toThrow(/line 3/);
|
||||
expect(getOrThrow(await fileSystem.readTextFile("session.jsonl", BACKGROUND_CONTEXT))).toBe(corrupted);
|
||||
});
|
||||
|
||||
it("rejects an unterminated header", async () => {
|
||||
const fileSystem = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
const options = { fileSystem, path: "session.jsonl", now: () => NOW };
|
||||
await fileSystem.writeFile("session.jsonl", JSON.stringify(header("torn")).slice(0, -4), BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(JsonlStorage.open(options, BACKGROUND_CONTEXT)).rejects.toThrow(/missing header/);
|
||||
});
|
||||
});
|
||||
2013
third_party/pi/packages/agent/test/harness/jsonl-v3-migration.test.ts
vendored
Normal file
2013
third_party/pi/packages/agent/test/harness/jsonl-v3-migration.test.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
94
third_party/pi/packages/agent/test/harness/memory-conformance.test.ts
vendored
Normal file
94
third_party/pi/packages/agent/test/harness/memory-conformance.test.ts
vendored
Normal file
@@ -0,0 +1,94 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import * as sessionWrites from "../../src/harness/session/commit.ts";
|
||||
import { MemorySessionRepo } from "../../src/harness/session/index.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import {
|
||||
type ConformanceCase,
|
||||
createSessionRepoConformance,
|
||||
createStorageConformance,
|
||||
type StorageFixture,
|
||||
} from "../../src/harness/session/testing/index.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
function registerConformance(name: string, cases: readonly ConformanceCase[]): void {
|
||||
describe(name, () => {
|
||||
for (const group of new Set(cases.map((testCase) => testCase.group))) {
|
||||
describe(group, () => {
|
||||
for (const testCase of cases.filter((candidate) => candidate.group === group)) {
|
||||
it(testCase.name, () => testCase.run());
|
||||
}
|
||||
});
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
registerConformance(
|
||||
"MemoryStorage conformance",
|
||||
createStorageConformance(() => {
|
||||
const storage = new MemoryStorage({ now: () => NOW });
|
||||
return Promise.resolve<StorageFixture>({
|
||||
storage,
|
||||
[Symbol.asyncDispose]: () => storage.close(BACKGROUND_CONTEXT),
|
||||
});
|
||||
}),
|
||||
);
|
||||
|
||||
describe("MemoryStorage commit statistics", () => {
|
||||
it("includes historical totals in the first commit after session reopen", async () => {
|
||||
const repo = new MemorySessionRepo({ now: () => NOW });
|
||||
const session = await repo.create({}, BACKGROUND_CONTEXT);
|
||||
const usage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 3,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
try {
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({
|
||||
id: "history",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "history", timestamp: NOW },
|
||||
}),
|
||||
sessionWrites.insertUsage({ id: "usage", usage, adjustment: false }),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
const reopened = await repo.open(session.metadata, BACKGROUND_CONTEXT);
|
||||
const result = await reopened.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit([storedValues.setValue(storedValues.sessionName, "reopened")], BACKGROUND_CONTEXT),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result.stats).toEqual({ messageCount: 1, usage });
|
||||
expect(result.stats).toEqual(await reopened.getStats(BACKGROUND_CONTEXT));
|
||||
await reopened.close(BACKGROUND_CONTEXT);
|
||||
} finally {
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
let memoryRepo: MemorySessionRepo;
|
||||
registerConformance(
|
||||
"MemorySessionRepo conformance",
|
||||
createSessionRepoConformance(
|
||||
() => {
|
||||
memoryRepo = new MemorySessionRepo({ now: () => NOW });
|
||||
return Promise.resolve(memoryRepo);
|
||||
},
|
||||
() => memoryRepo.close(BACKGROUND_CONTEXT),
|
||||
),
|
||||
);
|
||||
139
third_party/pi/packages/agent/test/harness/memory-session-repo.test.ts
vendored
Normal file
139
third_party/pi/packages/agent/test/harness/memory-session-repo.test.ts
vendored
Normal file
@@ -0,0 +1,139 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import * as sessionWrites from "../../src/harness/session/commit.ts";
|
||||
import { MemorySessionRepo } from "../../src/harness/session/index.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
function uuidTimestamp(id: string): number {
|
||||
return Number.parseInt(id.replaceAll("-", "").slice(0, 12), 16);
|
||||
}
|
||||
|
||||
describe("MemorySessionRepo metadata", () => {
|
||||
it("uses its injected clock for generated session identity and metadata", async () => {
|
||||
const repo = new MemorySessionRepo({ now: () => NOW });
|
||||
const session = await repo.create({}, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(session.metadata.createdAt).toBe(NOW);
|
||||
expect(uuidTimestamp(session.metadata.id)).toBe(NOW);
|
||||
await Promise.all([session.close(BACKGROUND_CONTEXT), repo.close(BACKGROUND_CONTEXT)]);
|
||||
});
|
||||
|
||||
it("returns a fresh facade after close while retaining one session and storage", async () => {
|
||||
const repo = new MemorySessionRepo({ now: () => NOW });
|
||||
const first = await repo.create({ id: "session" }, BACKGROUND_CONTEXT);
|
||||
const firstBranch = await first.createBranch("main", null, BACKGROUND_CONTEXT);
|
||||
const admittedWrite = first.setName("preserved", BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(repo.open(first.metadata, BACKGROUND_CONTEXT)).rejects.toThrow("already open");
|
||||
await Promise.all([admittedWrite, first.close(BACKGROUND_CONTEXT)]);
|
||||
await expect(first.getName(BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
await expect(first.scanBranch({ start: "entry" }, BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
await expect(firstBranch.getTipId(BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
|
||||
const second = await repo.open(first.metadata, BACKGROUND_CONTEXT);
|
||||
expect(second).not.toBe(first);
|
||||
expect(await second.getName(BACKGROUND_CONTEXT)).toBe("preserved");
|
||||
await second.close(BACKGROUND_CONTEXT);
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("waits for an explicit mutation before closing its facade", async () => {
|
||||
const repo = new MemorySessionRepo({ now: () => NOW });
|
||||
const session = await repo.create({ id: "session" }, BACKGROUND_CONTEXT);
|
||||
const mutation = await session.beginMutation(BACKGROUND_CONTEXT);
|
||||
let closed = false;
|
||||
const closing = session.close(BACKGROUND_CONTEXT).then(() => {
|
||||
closed = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(closed).toBe(false);
|
||||
await mutation.end(BACKGROUND_CONTEXT);
|
||||
await closing;
|
||||
expect(closed).toBe(true);
|
||||
|
||||
const reopened = await repo.open(session.metadata, BACKGROUND_CONTEXT);
|
||||
await Promise.all([reopened.close(BACKGROUND_CONTEXT), repo.close(BACKGROUND_CONTEXT)]);
|
||||
});
|
||||
|
||||
it("rejects an explicit scope that had not acquired before facade close", async () => {
|
||||
const repo = new MemorySessionRepo({ now: () => NOW });
|
||||
const session = await repo.create({ id: "session" }, BACKGROUND_CONTEXT);
|
||||
const first = await session.beginMutation(BACKGROUND_CONTEXT);
|
||||
const secondPromise = session.beginMutation(BACKGROUND_CONTEXT);
|
||||
void secondPromise.catch(() => {});
|
||||
let closed = false;
|
||||
const closing = session.close(BACKGROUND_CONTEXT).then(() => {
|
||||
closed = true;
|
||||
});
|
||||
|
||||
await first.end(BACKGROUND_CONTEXT);
|
||||
await expect(secondPromise).rejects.toThrow("Session is closed");
|
||||
await closing;
|
||||
expect(closed).toBe(true);
|
||||
|
||||
const reopened = await repo.open(session.metadata, BACKGROUND_CONTEXT);
|
||||
await Promise.all([reopened.close(BACKGROUND_CONTEXT), repo.close(BACKGROUND_CONTEXT)]);
|
||||
});
|
||||
|
||||
it("captures fork options before waiting for its snapshot boundary", async () => {
|
||||
const repo = new MemorySessionRepo({ now: () => NOW });
|
||||
const source = await repo.create({ id: "source" }, BACKGROUND_CONTEXT);
|
||||
const rootId = "00000000-0000-7000-8000-000000000001";
|
||||
const childId = "00000000-0000-7000-8000-000000000002";
|
||||
const mutation = await source.beginMutation(BACKGROUND_CONTEXT);
|
||||
const commit = mutation.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({
|
||||
id: rootId,
|
||||
parentId: null,
|
||||
type: "custom",
|
||||
customType: "root",
|
||||
}),
|
||||
sessionWrites.insertEntry({
|
||||
id: childId,
|
||||
parentId: rootId,
|
||||
type: "custom",
|
||||
customType: "child",
|
||||
}),
|
||||
storedValues.setValue(storedValues.branchTip("main"), childId),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), {
|
||||
model: { provider: "test", modelId: "test" },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const options = {
|
||||
id: "fork",
|
||||
scope: "branch" as const,
|
||||
branch: "main",
|
||||
entryId: childId,
|
||||
position: "before" as "before" | "at",
|
||||
};
|
||||
const fork = repo.fork(source.metadata, options, BACKGROUND_CONTEXT);
|
||||
options.entryId = rootId;
|
||||
options.position = "at";
|
||||
|
||||
await commit;
|
||||
await mutation.end(BACKGROUND_CONTEXT);
|
||||
await expect(
|
||||
source.scanBranch({ start: childId, order: "oldestFirst" }, BACKGROUND_CONTEXT),
|
||||
).resolves.toMatchObject([{ id: rootId }, { id: childId }]);
|
||||
const forked = await fork;
|
||||
const forkedMain = await forked.branch("main", BACKGROUND_CONTEXT);
|
||||
expect(await forkedMain?.getTipId(BACKGROUND_CONTEXT)).toBe(rootId);
|
||||
await Promise.all([
|
||||
source.close(BACKGROUND_CONTEXT),
|
||||
forked.close(BACKGROUND_CONTEXT),
|
||||
repo.close(BACKGROUND_CONTEXT),
|
||||
]);
|
||||
});
|
||||
});
|
||||
36
third_party/pi/packages/agent/test/harness/memory-storage.test.ts
vendored
Normal file
36
third_party/pi/packages/agent/test/harness/memory-storage.test.ts
vendored
Normal file
@@ -0,0 +1,36 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import * as sessionWrites from "../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
|
||||
describe("MemoryStorage", () => {
|
||||
it("uses the injected clock once per transaction", async () => {
|
||||
let timestamp = NOW;
|
||||
const storage = new MemoryStorage({ now: () => timestamp++ });
|
||||
|
||||
const first = await storage.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({
|
||||
id: "first",
|
||||
parentId: null,
|
||||
type: "custom",
|
||||
customType: "note",
|
||||
}),
|
||||
storedValues.setValue(storedValues.sessionName, "first"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const second = await storage.commit(
|
||||
[storedValues.setValue(storedValues.sessionName, "second")],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(first.timestamp).toBe(NOW);
|
||||
expect(second.timestamp).toBe(NOW + 1);
|
||||
expect((await storage.getEntries(["first"], BACKGROUND_CONTEXT)).get("first")?.timestamp).toBe(first.timestamp);
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
66
third_party/pi/packages/agent/test/harness/mutation-line.test.ts
vendored
Normal file
66
third_party/pi/packages/agent/test/harness/mutation-line.test.ts
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { MutationLine } from "../../src/harness/session/mutation-line.ts";
|
||||
|
||||
function deferred(): { promise: Promise<void>; resolve(): void } {
|
||||
let resolvePromise!: () => void;
|
||||
const promise = new Promise<void>((resolve) => {
|
||||
resolvePromise = resolve;
|
||||
});
|
||||
return { promise, resolve: resolvePromise };
|
||||
}
|
||||
|
||||
describe("MutationLine", () => {
|
||||
it("serializes every Session mutation", async () => {
|
||||
const line = new MutationLine();
|
||||
const gate = deferred();
|
||||
const order: string[] = [];
|
||||
const first = line.run(async () => {
|
||||
order.push("first:start");
|
||||
await gate.promise;
|
||||
order.push("first:end");
|
||||
return "first";
|
||||
});
|
||||
const second = line.run(() => {
|
||||
order.push("second");
|
||||
return "second";
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(order).toEqual(["first:start"]);
|
||||
gate.resolve();
|
||||
await expect(first).resolves.toBe("first");
|
||||
await expect(second).resolves.toBe("second");
|
||||
expect(order).toEqual(["first:start", "first:end", "second"]);
|
||||
});
|
||||
|
||||
it("continues after a failed job while preserving the original failure", async () => {
|
||||
const line = new MutationLine();
|
||||
const rejection = new Error("mutation failed");
|
||||
|
||||
await expect(
|
||||
line.run(() => {
|
||||
throw rejection;
|
||||
}),
|
||||
).rejects.toBe(rejection);
|
||||
await expect(line.run(() => "next")).resolves.toBe("next");
|
||||
});
|
||||
|
||||
it("seals queued and future jobs while draining the running job", async () => {
|
||||
const line = new MutationLine();
|
||||
const gate = deferred();
|
||||
const running = line.run(async () => {
|
||||
await gate.promise;
|
||||
return "running";
|
||||
});
|
||||
await Promise.resolve();
|
||||
const queued = line.run(() => "queued");
|
||||
const closed = new Error("closed");
|
||||
|
||||
const drained = line.seal(closed);
|
||||
await expect(line.run(() => "late")).rejects.toBe(closed);
|
||||
gate.resolve();
|
||||
await expect(running).resolves.toBe("running");
|
||||
await expect(queued).rejects.toBe(closed);
|
||||
await drained;
|
||||
});
|
||||
});
|
||||
706
third_party/pi/packages/agent/test/harness/nodejs-env.test.ts
vendored
Normal file
706
third_party/pi/packages/agent/test/harness/nodejs-env.test.ts
vendored
Normal file
@@ -0,0 +1,706 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { existsSync, readFileSync } from "node:fs";
|
||||
import { access, chmod, realpath, symlink } from "node:fs/promises";
|
||||
import { homedir } from "node:os";
|
||||
import { delimiter, join } from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT, withAbortSignal } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import { FileError, getOrThrow, type ShellExecOptions, type ShellOutputView } from "../../src/harness/types.ts";
|
||||
import { applyShellOutputUpdate } from "../../src/harness/utils/output-capture.ts";
|
||||
import { executeShellWithCapture } from "../../src/harness/utils/shell-output.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
const chmodRestorePaths: string[] = [];
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, ms: number, onTimeout?: () => void): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timeoutId = setTimeout(() => {
|
||||
onTimeout?.();
|
||||
reject(new Error(`Timed out after ${ms}ms`));
|
||||
}, ms);
|
||||
promise.then(
|
||||
(value) => {
|
||||
clearTimeout(timeoutId);
|
||||
resolve(value);
|
||||
},
|
||||
(error: unknown) => {
|
||||
clearTimeout(timeoutId);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async function collectShellOutput(
|
||||
env: NodeExecutionEnv,
|
||||
command: string,
|
||||
options: ShellExecOptions | undefined,
|
||||
context: Parameters<NodeExecutionEnv["exec"]>[2],
|
||||
): Promise<{ result: Awaited<ReturnType<NodeExecutionEnv["exec"]>>; output: ShellOutputView | undefined }> {
|
||||
let output: ShellOutputView | undefined;
|
||||
const result = await env.exec(
|
||||
command,
|
||||
{
|
||||
...options,
|
||||
onUpdate: (update) => {
|
||||
output = applyShellOutputUpdate(output, update);
|
||||
},
|
||||
},
|
||||
context,
|
||||
);
|
||||
return { result, output };
|
||||
}
|
||||
|
||||
function toBashSingleQuotedArg(value: string): string {
|
||||
return `'${value.replace(/\\/g, "/").replace(/'/g, `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function createInheritedStdioCommand(pidFile: string): string {
|
||||
return (
|
||||
'node -e "' +
|
||||
"const fs=require('fs');" +
|
||||
"const {spawn}=require('child_process');" +
|
||||
"const child=spawn(process.execPath,['-e','setTimeout(()=>{},60000)'],{stdio:'inherit',detached:true});" +
|
||||
"fs.writeFileSync(process.argv[1], String(child.pid));" +
|
||||
"child.unref();" +
|
||||
"console.log('child-exiting');" +
|
||||
'" ' +
|
||||
toBashSingleQuotedArg(pidFile)
|
||||
);
|
||||
}
|
||||
|
||||
function cleanupDetachedChild(pidFile: string): void {
|
||||
if (!existsSync(pidFile)) return;
|
||||
const pid = Number.parseInt(readFileSync(pidFile, "utf8").trim(), 10);
|
||||
if (!Number.isFinite(pid) || pid <= 0) return;
|
||||
try {
|
||||
execFileSync("taskkill", ["/F", "/T", "/PID", String(pid)], { stdio: "ignore" });
|
||||
} catch {}
|
||||
}
|
||||
|
||||
class FailingSpillExecutionEnv extends NodeExecutionEnv {
|
||||
override async createTempFile(
|
||||
options: Parameters<NodeExecutionEnv["createTempFile"]>[0],
|
||||
context: Parameters<NodeExecutionEnv["createTempFile"]>[1],
|
||||
) {
|
||||
if (options?.prefix === "pi-output-") {
|
||||
return { ok: true as const, value: join(this.cwd, "missing", "spill.log") };
|
||||
}
|
||||
return super.createTempFile(options, context);
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const path of chmodRestorePaths.splice(0)) {
|
||||
try {
|
||||
await access(path);
|
||||
await chmod(path, 0o700);
|
||||
} catch {}
|
||||
}
|
||||
});
|
||||
|
||||
describe("NodeExecutionEnv", () => {
|
||||
it("reads, writes, lists, and removes files and directories", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
expect(getOrThrow(await env.absolutePath("nested/child", BACKGROUND_CONTEXT))).toBe(join(root, "nested/child"));
|
||||
expect(getOrThrow(await env.joinPath([root, "nested", "child"], BACKGROUND_CONTEXT))).toBe(
|
||||
join(root, "nested", "child"),
|
||||
);
|
||||
getOrThrow(await env.createDir("nested/child", undefined, BACKGROUND_CONTEXT));
|
||||
getOrThrow(await env.writeFile("nested/child/file.txt", "hel", BACKGROUND_CONTEXT));
|
||||
getOrThrow(await env.appendFile("nested/child/file.txt", "lo", BACKGROUND_CONTEXT));
|
||||
expect(getOrThrow(await env.readTextFile("nested/child/file.txt", BACKGROUND_CONTEXT))).toBe("hello");
|
||||
expect(getOrThrow(await env.readTextLines("nested/child/file.txt", { maxLines: 1 }, BACKGROUND_CONTEXT))).toEqual(
|
||||
["hello"],
|
||||
);
|
||||
expect(
|
||||
Buffer.from(getOrThrow(await env.readBinaryFile("nested/child/file.txt", BACKGROUND_CONTEXT))).toString(
|
||||
"utf8",
|
||||
),
|
||||
).toBe("hello");
|
||||
|
||||
const entries = getOrThrow(await env.listDir("nested/child", BACKGROUND_CONTEXT));
|
||||
expect(entries).toHaveLength(1);
|
||||
expect(entries[0]).toMatchObject({
|
||||
name: "file.txt",
|
||||
path: join(root, "nested/child/file.txt"),
|
||||
kind: "file",
|
||||
size: 5,
|
||||
});
|
||||
expect(typeof entries[0]!.mtimeMs).toBe("number");
|
||||
|
||||
expect(getOrThrow(await env.exists("nested/child/file.txt", BACKGROUND_CONTEXT))).toBe(true);
|
||||
getOrThrow(await env.remove("nested/child/file.txt", undefined, BACKGROUND_CONTEXT));
|
||||
expect(getOrThrow(await env.exists("nested/child/file.txt", BACKGROUND_CONTEXT))).toBe(false);
|
||||
});
|
||||
|
||||
it("expands home-relative paths and file URLs", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
expect(getOrThrow(await env.absolutePath("~/pi-node-env-test", BACKGROUND_CONTEXT))).toBe(
|
||||
join(homedir(), "pi-node-env-test"),
|
||||
);
|
||||
const filePath = join(root, "file with spaces.txt");
|
||||
expect(getOrThrow(await env.absolutePath(pathToFileURL(filePath).href, BACKGROUND_CONTEXT))).toBe(filePath);
|
||||
});
|
||||
|
||||
it("returns fileInfo for files, directories, and symlinks without following symlinks", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.createDir("dir", { recursive: true }, BACKGROUND_CONTEXT));
|
||||
getOrThrow(await env.writeFile("dir/file.txt", "hello", BACKGROUND_CONTEXT));
|
||||
await symlink(join(root, "dir/file.txt"), join(root, "file-link"));
|
||||
await symlink(join(root, "dir"), join(root, "dir-link"));
|
||||
|
||||
expect(getOrThrow(await env.fileInfo("dir", BACKGROUND_CONTEXT))).toMatchObject({
|
||||
name: "dir",
|
||||
path: join(root, "dir"),
|
||||
kind: "directory",
|
||||
});
|
||||
expect(getOrThrow(await env.fileInfo("dir/file.txt", BACKGROUND_CONTEXT))).toMatchObject({
|
||||
name: "file.txt",
|
||||
path: join(root, "dir/file.txt"),
|
||||
kind: "file",
|
||||
size: 5,
|
||||
});
|
||||
expect(getOrThrow(await env.fileInfo("file-link", BACKGROUND_CONTEXT))).toMatchObject({
|
||||
name: "file-link",
|
||||
path: join(root, "file-link"),
|
||||
kind: "symlink",
|
||||
});
|
||||
expect(getOrThrow(await env.fileInfo("dir-link", BACKGROUND_CONTEXT))).toMatchObject({
|
||||
name: "dir-link",
|
||||
path: join(root, "dir-link"),
|
||||
kind: "symlink",
|
||||
});
|
||||
expect(getOrThrow(await env.canonicalPath("file-link", BACKGROUND_CONTEXT))).toBe(
|
||||
await realpath(join(root, "dir/file.txt")),
|
||||
);
|
||||
});
|
||||
|
||||
it("lists symlinks as symlinks", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile("target.txt", "hello", BACKGROUND_CONTEXT));
|
||||
await symlink(join(root, "target.txt"), join(root, "link.txt"));
|
||||
|
||||
const entries = getOrThrow(await env.listDir(".", BACKGROUND_CONTEXT));
|
||||
expect(
|
||||
entries.map((entry) => ({ name: entry.name, kind: entry.kind })).sort((a, b) => a.name.localeCompare(b.name)),
|
||||
).toEqual([
|
||||
{ name: "link.txt", kind: "symlink" },
|
||||
{ name: "target.txt", kind: "file" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("stops reading text lines at the requested limit", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile("file.txt", "one\ntwo\nthree", BACKGROUND_CONTEXT));
|
||||
expect(getOrThrow(await env.readTextLines("file.txt", { maxLines: 1 }, BACKGROUND_CONTEXT))).toEqual(["one"]);
|
||||
});
|
||||
|
||||
it("returns FileError for missing paths and keeps exists false for missing paths", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const info = await env.fileInfo("missing.txt", BACKGROUND_CONTEXT);
|
||||
expect(info.ok).toBe(false);
|
||||
if (!info.ok) {
|
||||
expect(info.error).toBeInstanceOf(FileError);
|
||||
expect(info.error).toMatchObject({
|
||||
name: "FileError",
|
||||
code: "not_found",
|
||||
path: join(root, "missing.txt"),
|
||||
});
|
||||
}
|
||||
expect(getOrThrow(await env.exists("missing.txt", BACKGROUND_CONTEXT))).toBe(false);
|
||||
});
|
||||
|
||||
it("returns FileError for listing non-directories", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile("file.txt", "hello", BACKGROUND_CONTEXT));
|
||||
const result = await env.listDir("file.txt", BACKGROUND_CONTEXT);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) {
|
||||
expect(result.error).toBeInstanceOf(FileError);
|
||||
expect(result.error).toMatchObject({ code: "not_directory" });
|
||||
}
|
||||
});
|
||||
|
||||
it("appends to new files and creates parent directories", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.appendFile("new/nested/file.txt", "a", BACKGROUND_CONTEXT));
|
||||
getOrThrow(await env.appendFile("new/nested/file.txt", "b", BACKGROUND_CONTEXT));
|
||||
expect(getOrThrow(await env.readTextFile("new/nested/file.txt", BACKGROUND_CONTEXT))).toBe("ab");
|
||||
});
|
||||
|
||||
it("atomically renames a file and replaces the destination", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile("source.txt", "new", BACKGROUND_CONTEXT));
|
||||
getOrThrow(await env.writeFile("destination.txt", "old", BACKGROUND_CONTEXT));
|
||||
|
||||
getOrThrow(await env.renameFile("source.txt", "destination.txt", BACKGROUND_CONTEXT));
|
||||
|
||||
expect(getOrThrow(await env.exists("source.txt", BACKGROUND_CONTEXT))).toBe(false);
|
||||
expect(getOrThrow(await env.readTextFile("destination.txt", BACKGROUND_CONTEXT))).toBe("new");
|
||||
});
|
||||
|
||||
it("reports the source path when rename fails because the source is missing", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile("destination.txt", "unchanged", BACKGROUND_CONTEXT));
|
||||
|
||||
const result = await env.renameFile("missing-source.txt", "destination.txt", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: {
|
||||
code: "not_found",
|
||||
path: join(root, "missing-source.txt"),
|
||||
},
|
||||
});
|
||||
expect(getOrThrow(await env.readTextFile("destination.txt", BACKGROUND_CONTEXT))).toBe("unchanged");
|
||||
});
|
||||
|
||||
it("creates temporary directories and files", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const tempDir = getOrThrow(await env.createTempDir("node-env-test-", BACKGROUND_CONTEXT));
|
||||
await expect(access(tempDir)).resolves.toBeUndefined();
|
||||
const tempFile = getOrThrow(await env.createTempFile({ prefix: "prefix-", suffix: ".txt" }, BACKGROUND_CONTEXT));
|
||||
await expect(access(tempFile)).resolves.toBeUndefined();
|
||||
expect(tempFile.endsWith(".txt")).toBe(true);
|
||||
});
|
||||
|
||||
it("honors createDir recursive false and remove recursive/force options", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const createResult = await env.createDir("missing/child", { recursive: false }, BACKGROUND_CONTEXT);
|
||||
expect(createResult.ok).toBe(false);
|
||||
if (!createResult.ok) expect(createResult.error).toMatchObject({ code: "not_found" });
|
||||
|
||||
getOrThrow(await env.writeFile("dir/child/file.txt", "hello", BACKGROUND_CONTEXT));
|
||||
const removeDirectory = await env.remove("dir", { recursive: false }, BACKGROUND_CONTEXT);
|
||||
expect(removeDirectory.ok).toBe(false);
|
||||
getOrThrow(await env.remove("dir", { recursive: true }, BACKGROUND_CONTEXT));
|
||||
expect(getOrThrow(await env.exists("dir", BACKGROUND_CONTEXT))).toBe(false);
|
||||
|
||||
const removeMissing = await env.remove("missing", { force: false }, BACKGROUND_CONTEXT);
|
||||
expect(removeMissing.ok).toBe(false);
|
||||
getOrThrow(await env.remove("missing", { force: true }, BACKGROUND_CONTEXT));
|
||||
});
|
||||
|
||||
it("returns aborted results for pre-aborted cancellable file operations", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile("file.txt", "hello", BACKGROUND_CONTEXT));
|
||||
const controller = new AbortController();
|
||||
controller.abort();
|
||||
const context = withAbortSignal(controller.signal, BACKGROUND_CONTEXT);
|
||||
|
||||
const results = await Promise.all([
|
||||
env.readTextFile("file.txt", context),
|
||||
env.readTextLines("file.txt", undefined, context),
|
||||
env.readBinaryFile("file.txt", context),
|
||||
env.writeFile("other.txt", "hello", context),
|
||||
env.renameFile("file.txt", "renamed.txt", context),
|
||||
env.listDir(".", context),
|
||||
]);
|
||||
for (const result of results) {
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.error).toMatchObject({ code: "aborted" });
|
||||
}
|
||||
});
|
||||
|
||||
it("cleanup is best-effort", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await expect(env.cleanup(BACKGROUND_CONTEXT)).resolves.toBeUndefined();
|
||||
});
|
||||
|
||||
it("executes commands in cwd with env overrides", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const collected = await collectShellOutput(
|
||||
env,
|
||||
'printf \'%s:%s\' "$PWD" "$NODE_ENV_TEST"',
|
||||
{ env: { NODE_ENV_TEST: "ok" } },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const result = getOrThrow(collected.result);
|
||||
expect(collected.output?.text).toBe(`${await realpath(root)}:ok`);
|
||||
expect(result.exitCode).toBe(0);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a missing override preserves the base value", undefined, "x:/stale/parent.jsonl"],
|
||||
["an empty override shadows the base value", { PI_SESSION_FILE: "" }, "x:"],
|
||||
[
|
||||
"a string override replaces the base value",
|
||||
{ PI_SESSION_FILE: "/sessions/current.jsonl" },
|
||||
"x:/sessions/current.jsonl",
|
||||
],
|
||||
] as const)(
|
||||
"applies string shell environment overrides when %s",
|
||||
async (_description, overrides, expectedSessionFile) => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({
|
||||
cwd: root,
|
||||
shellEnv: {
|
||||
PI_SESSION_FILE: "/stale/parent.jsonl",
|
||||
PI_CODING_AGENT: "true",
|
||||
PI_NODE_ENV_PRESERVED_TEST: "preserved",
|
||||
},
|
||||
});
|
||||
const collected = await collectShellOutput(
|
||||
env,
|
||||
`printf '%s:%s|%s|%s' "\${PI_SESSION_FILE+x}" "\${PI_SESSION_FILE-}" "$PI_CODING_AGENT" "$PI_NODE_ENV_PRESERVED_TEST"`,
|
||||
{ env: overrides },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
getOrThrow(collected.result);
|
||||
expect(collected.output?.text).toBe(`${expectedSessionFile}|true|preserved`);
|
||||
},
|
||||
);
|
||||
|
||||
it("can replace rather than inherit the default shell environment", async () => {
|
||||
const root = createTempDir();
|
||||
const inheritedKey = "PI_NODE_ENV_INHERITED_TEST";
|
||||
const configuredKey = "PI_NODE_ENV_CONFIGURED_TEST";
|
||||
const explicitKey = "PI_NODE_ENV_EXPLICIT_TEST";
|
||||
const previousInherited = process.env[inheritedKey];
|
||||
process.env[inheritedKey] = "host";
|
||||
try {
|
||||
const env = new NodeExecutionEnv({ cwd: root, shellEnv: { [configuredKey]: "configured" } });
|
||||
const collected = await collectShellOutput(
|
||||
env,
|
||||
`printf '%s:%s:%s' "\${${inheritedKey}-}" "\${${configuredKey}-}" "\${${explicitKey}-}"`,
|
||||
{ inheritEnv: false, env: { [explicitKey]: "explicit" } },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
getOrThrow(collected.result);
|
||||
expect(collected.output?.text).toBe("::explicit");
|
||||
} finally {
|
||||
if (previousInherited === undefined) delete process.env[inheritedKey];
|
||||
else process.env[inheritedKey] = previousInherited;
|
||||
}
|
||||
});
|
||||
|
||||
it("uses stdin command transport for legacy WSL bash paths", async () => {
|
||||
if (process.platform === "win32") return;
|
||||
const root = createTempDir();
|
||||
const shellPath = "C:\\Windows\\System32\\bash.exe";
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(
|
||||
await env.writeFile(
|
||||
shellPath,
|
||||
'#!/bin/sh\nprintf \'args:%s\\n\' "$*" >&2\nexec /bin/bash "$@"\n',
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
await chmod(join(root, shellPath), 0o755);
|
||||
|
||||
const originalCwd = process.cwd();
|
||||
const originalPath = process.env.PATH;
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
try {
|
||||
process.chdir(root);
|
||||
process.env.PATH = `${root}${delimiter}${originalPath ?? ""}`;
|
||||
Object.defineProperty(process, "platform", {
|
||||
configurable: true,
|
||||
value: "win32",
|
||||
});
|
||||
|
||||
const wslEnv = new NodeExecutionEnv({ cwd: root, shellPath });
|
||||
const nameExpansion = "$" + "{name}";
|
||||
const collected = await collectShellOutput(
|
||||
wslEnv,
|
||||
`name='World'; echo "Hello, ${nameExpansion}!"`,
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const result = getOrThrow(collected.result);
|
||||
expect(collected.output?.text).toContain("Hello, World!");
|
||||
expect(collected.output?.text).toContain("args:-s");
|
||||
expect(result.exitCode).toBe(0);
|
||||
} finally {
|
||||
process.chdir(originalCwd);
|
||||
process.env.PATH = originalPath;
|
||||
if (platformDescriptor) {
|
||||
Object.defineProperty(process, "platform", platformDescriptor);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it.skipIf(process.platform !== "win32")(
|
||||
"settles after the shell exits when a detached descendant retains inherited stdio",
|
||||
async () => {
|
||||
const root = createTempDir();
|
||||
const pidFile = join(root, "grandchild.pid");
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const controller = new AbortController();
|
||||
try {
|
||||
const collected = await withTimeout(
|
||||
collectShellOutput(
|
||||
env,
|
||||
createInheritedStdioCommand(pidFile),
|
||||
undefined,
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
),
|
||||
3000,
|
||||
() => controller.abort(),
|
||||
);
|
||||
getOrThrow(collected.result);
|
||||
expect(collected.output?.text).toContain("child-exiting");
|
||||
} finally {
|
||||
controller.abort();
|
||||
cleanupDetachedChild(pidFile);
|
||||
}
|
||||
},
|
||||
);
|
||||
|
||||
it("cleanup terminates active shell processes", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const execution = env.exec("touch started; sleep 60", undefined, BACKGROUND_CONTEXT);
|
||||
for (let attempt = 0; attempt < 100 && !getOrThrow(await env.exists("started", BACKGROUND_CONTEXT)); attempt++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(getOrThrow(await env.exists("started", BACKGROUND_CONTEXT))).toBe(true);
|
||||
await env.cleanup(BACKGROUND_CONTEXT);
|
||||
await expect(withTimeout(execution, 3000)).resolves.toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it("combines stdout and stderr into one bounded view", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const updates: string[] = [];
|
||||
let output: ShellOutputView | undefined;
|
||||
const result = getOrThrow(
|
||||
await env.exec(
|
||||
"printf out; printf err >&2",
|
||||
{
|
||||
onUpdate: (update) => {
|
||||
updates.push(update.kind);
|
||||
output = applyShellOutputUpdate(output, update);
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(result.exitCode).toBe(0);
|
||||
expect(output?.text).toContain("out");
|
||||
expect(output?.text).toContain("err");
|
||||
expect(updates[0]).toBe("replace");
|
||||
});
|
||||
|
||||
it("reports a missing working directory before spawning", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: join(root, "missing") });
|
||||
const result = await env.exec("printf ok", undefined, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "spawn_error", message: expect.stringContaining("Working directory does not exist") },
|
||||
});
|
||||
});
|
||||
|
||||
it("returns non-zero command exit codes as successful execution results", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const result = getOrThrow(await env.exec("exit 7", undefined, BACKGROUND_CONTEXT));
|
||||
expect(result.exitCode).toBe(7);
|
||||
expect(result.truncation.totalBytes).toBe(0);
|
||||
});
|
||||
|
||||
// Regression test for https://github.com/earendil-works/pi/issues/8992
|
||||
it.skipIf(process.platform === "win32")("maps signal-killed processes to a non-zero exit code", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const result = getOrThrow(await env.exec("kill -9 $$", undefined, BACKGROUND_CONTEXT));
|
||||
expect(result.exitCode).toBe(128 + 9);
|
||||
});
|
||||
|
||||
it("returns timeout errors for commands exceeding the timeout", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const result = await env.exec("sleep 5", { timeout: 0.01 }, BACKGROUND_CONTEXT);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.error).toMatchObject({ code: "timeout" });
|
||||
});
|
||||
|
||||
it("returns callback errors from exec stream handlers", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const result = await env.exec(
|
||||
"printf out",
|
||||
{
|
||||
onUpdate: () => {
|
||||
throw new Error("callback failed");
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.error).toMatchObject({ code: "callback_error", message: "callback failed" });
|
||||
});
|
||||
|
||||
it("returns shell unavailable and spawn errors", async () => {
|
||||
const root = createTempDir();
|
||||
const missingShellEnv = new NodeExecutionEnv({ cwd: root, shellPath: join(root, "missing-shell") });
|
||||
const missingShell = await missingShellEnv.exec("printf ok", undefined, BACKGROUND_CONTEXT);
|
||||
expect(missingShell.ok).toBe(false);
|
||||
if (!missingShell.ok) expect(missingShell.error).toMatchObject({ code: "shell_unavailable" });
|
||||
|
||||
const shellPath = join(root, "not-executable-shell");
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
getOrThrow(await env.writeFile(shellPath, "not executable", BACKGROUND_CONTEXT));
|
||||
const spawnErrorEnv = new NodeExecutionEnv({ cwd: root, shellPath });
|
||||
const spawnError = await spawnErrorEnv.exec("printf ok", undefined, BACKGROUND_CONTEXT);
|
||||
expect(spawnError.ok).toBe(false);
|
||||
if (!spawnError.ok) expect(spawnError.error).toMatchObject({ code: "spawn_error" });
|
||||
});
|
||||
|
||||
it("returns an aborted result for aborted commands", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const controller = new AbortController();
|
||||
const promise = env.exec("sleep 5", undefined, withAbortSignal(controller.signal, BACKGROUND_CONTEXT));
|
||||
controller.abort();
|
||||
const result = await promise;
|
||||
expect(result.ok).toBe(false);
|
||||
if (!result.ok) expect(result.error).toMatchObject({ code: "aborted" });
|
||||
});
|
||||
|
||||
it.skipIf(process.platform === "win32")("ignores asynchronous taskkill spawn errors during abort", async () => {
|
||||
const root = createTempDir();
|
||||
const pidFile = join(root, "shell.pid");
|
||||
const controller = new AbortController();
|
||||
const env = new NodeExecutionEnv({ cwd: root, shellPath: "/bin/bash" });
|
||||
const platformDescriptor = Object.getOwnPropertyDescriptor(process, "platform");
|
||||
const previousSystemRoot = process.env.SystemRoot;
|
||||
process.env.SystemRoot = "/definitely/missing/windows";
|
||||
Object.defineProperty(process, "platform", { configurable: true, value: "win32" });
|
||||
|
||||
let pid: number | undefined;
|
||||
try {
|
||||
const execution = env.exec(
|
||||
`echo $$ > ${toBashSingleQuotedArg(pidFile)}; exec sleep 60`,
|
||||
undefined,
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
);
|
||||
for (let attempt = 0; attempt < 100 && !existsSync(pidFile); attempt++) {
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
expect(existsSync(pidFile)).toBe(true);
|
||||
|
||||
controller.abort();
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
pid = Number.parseInt(readFileSync(pidFile, "utf8"), 10);
|
||||
process.kill(pid, "SIGKILL");
|
||||
|
||||
const result = await execution;
|
||||
expect(result).toMatchObject({ ok: false, error: { code: "aborted" } });
|
||||
} finally {
|
||||
if (pid === undefined && existsSync(pidFile)) {
|
||||
pid = Number.parseInt(readFileSync(pidFile, "utf8"), 10);
|
||||
}
|
||||
if (pid !== undefined && Number.isFinite(pid)) {
|
||||
try {
|
||||
process.kill(pid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
if (previousSystemRoot === undefined) delete process.env.SystemRoot;
|
||||
else process.env.SystemRoot = previousSystemRoot;
|
||||
if (platformDescriptor) Object.defineProperty(process, "platform", platformDescriptor);
|
||||
}
|
||||
});
|
||||
|
||||
it("does not create a spill before bounded output crosses its limits", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const result = getOrThrow(
|
||||
await env.exec(
|
||||
"printf short",
|
||||
{
|
||||
capture: { limits: { maxBytes: 100, maxLines: 10, retain: "tail" }, spill: true },
|
||||
onUpdate: () => {},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(result.spillPath).toBeUndefined();
|
||||
});
|
||||
|
||||
it("preserves exact raw bytes in the spill while decoding a bounded text view", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const expected = [0x66, 0x80, 0x00, 0x6f];
|
||||
const result = getOrThrow(
|
||||
await env.exec(
|
||||
`${JSON.stringify(process.execPath)} -e "process.stdout.write(Buffer.from([${expected.join(",")}]))"`,
|
||||
{
|
||||
capture: { limits: { maxBytes: 1, maxLines: 10, retain: "tail" }, spill: true },
|
||||
onUpdate: () => {},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(result.spillPath).toBeDefined();
|
||||
expect([...getOrThrow(await env.readBinaryFile(result.spillPath!, BACKGROUND_CONTEXT))]).toEqual(expected);
|
||||
});
|
||||
|
||||
it("fails rather than silently losing a requested spill", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new FailingSpillExecutionEnv({ cwd: root });
|
||||
const result = await env.exec(
|
||||
"printf 12345678901234567890",
|
||||
{
|
||||
capture: { limits: { maxBytes: 10, maxLines: 10, retain: "tail" }, spill: true },
|
||||
onUpdate: () => {},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(result).toMatchObject({
|
||||
ok: false,
|
||||
error: { code: "unknown", message: expect.stringContaining("Failed to preserve complete shell output") },
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves complete output when spill-stream backpressure pauses a process that exits quickly", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const size = 500_000;
|
||||
const result = getOrThrow(
|
||||
await env.exec(
|
||||
`${JSON.stringify(process.execPath)} -e "process.stdout.write('x'.repeat(${size}))"`,
|
||||
{
|
||||
capture: { limits: { maxBytes: 10, maxLines: 10, retain: "tail" }, spill: true },
|
||||
onUpdate: () => {},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
expect(result.spillPath).toBeDefined();
|
||||
expect(getOrThrow(await env.readTextFile(result.spillPath!, BACKGROUND_CONTEXT))).toHaveLength(size);
|
||||
});
|
||||
|
||||
it("captures large shell output to a full output file through the execution env", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
const result = getOrThrow(
|
||||
await executeShellWithCapture(env, "yes line | head -n 15000", undefined, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.fullOutputPath).toBeDefined();
|
||||
const fullOutput = getOrThrow(await env.readTextFile(result.fullOutputPath!, BACKGROUND_CONTEXT));
|
||||
expect(fullOutput.split("\n").length).toBeGreaterThan(10000);
|
||||
expect(result.output.length).toBeLessThan(fullOutput.length);
|
||||
});
|
||||
});
|
||||
151
third_party/pi/packages/agent/test/harness/output-capture.test.ts
vendored
Normal file
151
third_party/pi/packages/agent/test/harness/output-capture.test.ts
vendored
Normal file
@@ -0,0 +1,151 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import type { ShellOutputUpdate, ShellOutputView } from "../../src/harness/types.ts";
|
||||
import { applyShellOutputUpdate, OutputCapture, sanitizeShellOutput } from "../../src/harness/utils/output-capture.ts";
|
||||
|
||||
function createCapture(options?: { maxBytes?: number; maxLines?: number; retain?: "head" | "tail" }) {
|
||||
const updates: ShellOutputUpdate[] = [];
|
||||
const errors: unknown[] = [];
|
||||
const capture = new OutputCapture(
|
||||
{
|
||||
limits: {
|
||||
maxBytes: options?.maxBytes ?? 50,
|
||||
maxLines: options?.maxLines ?? 100,
|
||||
retain: options?.retain ?? "tail",
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
{
|
||||
onUpdate: (update) => updates.push(update),
|
||||
onError: (error) => errors.push(error),
|
||||
},
|
||||
);
|
||||
return { capture, updates, errors };
|
||||
}
|
||||
|
||||
function fold(updates: ShellOutputUpdate[]): ShellOutputView | undefined {
|
||||
let output: ShellOutputView | undefined;
|
||||
for (const update of updates) output = applyShellOutputUpdate(output, update);
|
||||
return output;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
});
|
||||
|
||||
describe("OutputCapture", () => {
|
||||
it("removes invalid control characters without changing text or line boundaries", () => {
|
||||
const input = "a\0b\tc\nd\re\u0007f\ufff9g\ufffbh😀";
|
||||
expect(sanitizeShellOutput(input)).toBe("ab\tc\ndefgh😀");
|
||||
const { capture } = createCapture();
|
||||
capture.push(input);
|
||||
expect(capture.snapshot().text).toBe("ab\tc\ndefgh😀");
|
||||
});
|
||||
it("decodes UTF-8 split across raw process chunks", () => {
|
||||
const { capture } = createCapture();
|
||||
const bytes = new TextEncoder().encode("😀");
|
||||
capture.push(bytes.subarray(0, 2));
|
||||
expect(capture.snapshot().text).toBe("");
|
||||
capture.push(bytes.subarray(2));
|
||||
capture.finish();
|
||||
expect(capture.snapshot().text).toBe("😀");
|
||||
});
|
||||
|
||||
it("publishes the first bounded view immediately and trickling appends responsively", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
const { capture, updates } = createCapture();
|
||||
|
||||
capture.push("one");
|
||||
expect(updates).toHaveLength(1);
|
||||
expect(updates[0]?.kind).toBe("replace");
|
||||
|
||||
vi.advanceTimersByTime(150);
|
||||
capture.push(" two");
|
||||
expect(updates).toHaveLength(2);
|
||||
expect(updates[1]).toMatchObject({ kind: "append", text: " two" });
|
||||
expect(fold(updates)?.text).toBe("one two");
|
||||
});
|
||||
|
||||
it("collapses a burst into one trailing update", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
const { capture, updates } = createCapture();
|
||||
capture.push("a");
|
||||
capture.push("b");
|
||||
capture.push("c");
|
||||
expect(updates).toHaveLength(1);
|
||||
|
||||
vi.advanceTimersByTime(100);
|
||||
expect(updates).toHaveLength(2);
|
||||
expect(updates[1]).toMatchObject({ kind: "append", text: "bc" });
|
||||
expect(fold(updates)?.text).toBe("abc");
|
||||
});
|
||||
|
||||
it("publishes a small slide for post-cap trickle", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
const { capture, updates } = createCapture({ maxBytes: 10 });
|
||||
capture.push("abcdefghij");
|
||||
vi.advanceTimersByTime(150);
|
||||
capture.push("k");
|
||||
|
||||
expect(updates[1]).toMatchObject({ kind: "slide", drop: 1, text: "k" });
|
||||
expect(fold(updates)?.text).toBe("bcdefghijk");
|
||||
expect(fold(updates)?.truncation.totalBytes).toBe(11);
|
||||
});
|
||||
|
||||
it("keeps the exact byte count for a single line larger than its working buffer", () => {
|
||||
vi.useFakeTimers();
|
||||
const { capture } = createCapture({ maxBytes: 10 });
|
||||
capture.push("x".repeat(100));
|
||||
expect(capture.snapshot()).toMatchObject({
|
||||
text: "x".repeat(10),
|
||||
lastLineBytes: 100,
|
||||
truncation: { lastLinePartial: true },
|
||||
});
|
||||
});
|
||||
|
||||
it("uses a cap-bounded replacement after complete turnover", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
const { capture, updates } = createCapture({ maxBytes: 10 });
|
||||
capture.push("abcdefghij");
|
||||
capture.push("x".repeat(100));
|
||||
vi.advanceTimersByTime(100);
|
||||
|
||||
expect(updates[1]?.kind).toBe("replace");
|
||||
expect(fold(updates)?.text).toHaveLength(10);
|
||||
expect(fold(updates)?.truncation.totalBytes).toBe(110);
|
||||
});
|
||||
|
||||
it("forces held state and cancels its trailing timer on dispose", () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(0);
|
||||
const { capture, updates } = createCapture();
|
||||
capture.push("a");
|
||||
capture.push("b");
|
||||
capture.flush();
|
||||
expect(fold(updates)?.text).toBe("ab");
|
||||
capture.dispose();
|
||||
vi.advanceTimersByTime(1_000);
|
||||
expect(updates).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("preserves the original head after its raw guard is crossed", () => {
|
||||
vi.useFakeTimers();
|
||||
const { capture } = createCapture({ maxBytes: 100, maxLines: 2, retain: "head" });
|
||||
capture.push(`first\nsecond\n${"tail".repeat(100)}`);
|
||||
expect(capture.snapshot().text).toBe("first\nsecond");
|
||||
});
|
||||
|
||||
it("publishes spill metadata without resending text", () => {
|
||||
vi.useFakeTimers();
|
||||
const { capture, updates, errors } = createCapture();
|
||||
capture.push("output");
|
||||
capture.setSpillPath("/tmp/output.log");
|
||||
expect(updates.at(-1)).toMatchObject({ kind: "metadata", metadata: { spillPath: "/tmp/output.log" } });
|
||||
expect(fold(updates)?.spillPath).toBe("/tmp/output.log");
|
||||
expect(errors).toEqual([]);
|
||||
});
|
||||
});
|
||||
97
third_party/pi/packages/agent/test/harness/prompt-templates.test.ts
vendored
Normal file
97
third_party/pi/packages/agent/test/harness/prompt-templates.test.ts
vendored
Normal file
@@ -0,0 +1,97 @@
|
||||
import { symlink } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import {
|
||||
formatPromptTemplateInvocation,
|
||||
loadPromptTemplates,
|
||||
loadSourcedPromptTemplates,
|
||||
} from "../../src/harness/prompt-templates.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
describe("loadPromptTemplates", () => {
|
||||
it("loads markdown templates non-recursively from one or more dirs", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("a/nested", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.createDir("b", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile("a/one.md", "---\ndescription: One template\n---\nHello $1", BACKGROUND_CONTEXT);
|
||||
await env.writeFile("a/nested/ignored.md", "Ignored", BACKGROUND_CONTEXT);
|
||||
await env.writeFile("b/two.md", "First line description\nBody", BACKGROUND_CONTEXT);
|
||||
|
||||
const { promptTemplates, diagnostics } = await loadPromptTemplates(env, ["a", "b"], BACKGROUND_CONTEXT);
|
||||
|
||||
expect(diagnostics).toEqual([]);
|
||||
expect(promptTemplates).toEqual([
|
||||
{ name: "one", description: "One template", content: "Hello $1" },
|
||||
{ name: "two", description: "First line description", content: "First line description\nBody" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("preserves source info for sourced prompt templates", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("prompts", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile("prompts/example.md", "---\ndescription: Example\n---\nExample body", BACKGROUND_CONTEXT);
|
||||
|
||||
const { promptTemplates, diagnostics } = await loadSourcedPromptTemplates(
|
||||
env,
|
||||
[{ path: "prompts", source: { type: "project" as const } }],
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(diagnostics).toEqual([]);
|
||||
expect(promptTemplates).toEqual([
|
||||
{
|
||||
promptTemplate: { name: "example", description: "Example", content: "Example body" },
|
||||
source: { type: "project" },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("attaches source info to diagnostics", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.writeFile("broken.md", "---\ndescription: [unterminated\n---\nBody", BACKGROUND_CONTEXT);
|
||||
|
||||
const { promptTemplates, diagnostics } = await loadSourcedPromptTemplates(
|
||||
env,
|
||||
[{ path: "broken.md", source: { type: "user" as const } }],
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(promptTemplates).toEqual([]);
|
||||
expect(diagnostics).toHaveLength(1);
|
||||
expect(diagnostics[0]).toMatchObject({
|
||||
type: "warning",
|
||||
path: join(root, "broken.md"),
|
||||
source: { type: "user" },
|
||||
});
|
||||
});
|
||||
|
||||
it("loads explicit markdown files and symlinked files", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.writeFile("target.md", "---\ndescription: Target\n---\nTarget body", BACKGROUND_CONTEXT);
|
||||
await symlink(join(root, "target.md"), join(root, "link.md"));
|
||||
|
||||
const { promptTemplates } = await loadPromptTemplates(env, ["target.md", "link.md"], BACKGROUND_CONTEXT);
|
||||
|
||||
expect(promptTemplates).toEqual([
|
||||
{ name: "target", description: "Target", content: "Target body" },
|
||||
{ name: "link", description: "Target", content: "Target body" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("formatPromptTemplateInvocation", () => {
|
||||
it("substitutes command arguments", () => {
|
||||
const content = "$1 $" + "{@:2} $ARGUMENTS";
|
||||
expect(formatPromptTemplateInvocation({ name: "one", content }, ["hello world", "test"])).toBe(
|
||||
"hello world test hello world test",
|
||||
);
|
||||
});
|
||||
});
|
||||
24
third_party/pi/packages/agent/test/harness/resource-formatting.test.ts
vendored
Normal file
24
third_party/pi/packages/agent/test/harness/resource-formatting.test.ts
vendored
Normal file
@@ -0,0 +1,24 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatPromptTemplateInvocation } from "../../src/harness/prompt-templates.ts";
|
||||
import { formatSkillInvocation } from "../../src/harness/skills.ts";
|
||||
|
||||
describe("resource formatting helpers", () => {
|
||||
it("formats skill invocations with additional instructions", () => {
|
||||
const skill = {
|
||||
name: "inspect",
|
||||
description: "Inspect things",
|
||||
content: "Use inspection tools.",
|
||||
filePath: "/project/.pi/skills/inspect/SKILL.md",
|
||||
};
|
||||
|
||||
expect(formatSkillInvocation(skill, "Check errors.")).toBe(
|
||||
'<skill name="inspect" location="/project/.pi/skills/inspect/SKILL.md">\nReferences are relative to /project/.pi/skills/inspect.\n\nUse inspection tools.\n</skill>\n\nCheck errors.',
|
||||
);
|
||||
});
|
||||
|
||||
it("formats prompt template invocations with positional arguments", () => {
|
||||
expect(
|
||||
formatPromptTemplateInvocation({ name: "review", content: "Review $1 with $ARGUMENTS" }, ["a.ts", "care"]),
|
||||
).toBe("Review a.ts with a.ts care");
|
||||
});
|
||||
});
|
||||
677
third_party/pi/packages/agent/test/harness/runtime/accept.test.ts
vendored
Normal file
677
third_party/pi/packages/agent/test/harness/runtime/accept.test.ts
vendored
Normal file
@@ -0,0 +1,677 @@
|
||||
import { createModels, fauxAssistantMessage, fauxProvider } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HarnessEvent, HarnessFault, type OperationRequest } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT, createContextKey, withContextValue } from "../../../src/harness/context.ts";
|
||||
import type { Result } from "../../../src/harness/result.ts";
|
||||
import { createAgentHarness, Harness } from "../../../src/harness/runtime/harness.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import { InstrumentedStorage } from "../../../src/harness/session/testing/index.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import type { AgentMessage, Session } from "../../../src/index.ts";
|
||||
import { ControlledMemoryStorage, deferred, FailingMemoryStorage } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const configuration = {
|
||||
model: { provider: "faux", modelId: "faux-1" },
|
||||
thinkingLevel: "off" as const,
|
||||
activeToolNames: [],
|
||||
};
|
||||
|
||||
async function createSession(storage = new InstrumentedStorage(new MemoryStorage())): Promise<{
|
||||
session: Session;
|
||||
storage: InstrumentedStorage;
|
||||
}> {
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `accept-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return { session, storage };
|
||||
}
|
||||
|
||||
function options(session: Session) {
|
||||
const provider = fauxProvider();
|
||||
const models = createModels();
|
||||
models.setProvider(provider.provider);
|
||||
return { session, models, model: provider.getModel() };
|
||||
}
|
||||
|
||||
async function createHarness(
|
||||
beforeCreate?: (session: Session) => Promise<void>,
|
||||
resources: Parameters<typeof createAgentHarness>[0]["resources"] = {},
|
||||
queueModes: { steeringMode?: "all" | "one-at-a-time"; followUpMode?: "all" | "one-at-a-time" } = {},
|
||||
): Promise<{
|
||||
harness: Harness<object | undefined>;
|
||||
lane: Lane<object | undefined>;
|
||||
session: Session;
|
||||
storage: InstrumentedStorage;
|
||||
models: ReturnType<typeof createModels>;
|
||||
}> {
|
||||
const created = await createSession();
|
||||
await beforeCreate?.(created.session);
|
||||
const harnessOptions = { ...options(created.session), resources, ...queueModes };
|
||||
const { harness } = await createAgentHarness(harnessOptions, BACKGROUND_CONTEXT);
|
||||
if (!(harness instanceof Harness)) throw new Error("Expected runtime Harness");
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
if (!(lane instanceof Lane)) throw new Error("Expected runtime Lane");
|
||||
created.storage.clearCommitAttempts();
|
||||
return { harness, lane, ...created, models: harnessOptions.models };
|
||||
}
|
||||
|
||||
function unwrap<T>(result: Result<T, unknown>): T {
|
||||
if (!result.ok) throw result.error;
|
||||
return result.value;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime atomic run acceptance", () => {
|
||||
it.each([
|
||||
["text", { kind: "prompt", prompt: "hello" } satisfies OperationRequest, [{ type: "text", text: "hello" }]],
|
||||
[
|
||||
"images",
|
||||
{
|
||||
kind: "prompt",
|
||||
prompt: "",
|
||||
images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
|
||||
} satisfies OperationRequest,
|
||||
[{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
|
||||
],
|
||||
[
|
||||
"text and images",
|
||||
{
|
||||
kind: "prompt",
|
||||
prompt: "hello",
|
||||
images: [{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" }],
|
||||
} satisfies OperationRequest,
|
||||
[
|
||||
{ type: "text", text: "hello" },
|
||||
{ type: "image", data: "aW1hZ2U=", mimeType: "image/png" },
|
||||
],
|
||||
],
|
||||
] as const)("accepts normalized %s prompts into starting", async (_name, request, expectedContent) => {
|
||||
const { lane, session } = await createHarness();
|
||||
|
||||
const admission = unwrap(await lane.accept(request, BACKGROUND_CONTEXT));
|
||||
const operation = lane.state.operation;
|
||||
if (operation?.state.at !== "starting") throw new Error("Expected accepted run");
|
||||
const entryId = operation.meta.intent.kind === "run" ? operation.meta.intent.promptEntryIds[0] : undefined;
|
||||
if (entryId === undefined) throw new Error("Expected prompt entry");
|
||||
const entry = await session.getEntry(entryId, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(admission).toMatchObject({ operationId: operation.meta.operationId, kind: "run" });
|
||||
expect(entry).toMatchObject({ type: "message", message: { role: "user", content: expectedContent } });
|
||||
expect(operation.state.settings).toEqual({
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
});
|
||||
});
|
||||
|
||||
it("preserves supplied message arrays and commits the exact acceptance write families once", async () => {
|
||||
const { harness, lane, storage, models } = await createHarness();
|
||||
const messages: AgentMessage[] = [
|
||||
{ role: "user", content: "one", timestamp: 10 },
|
||||
{ role: "user", content: "two", timestamp: 11 },
|
||||
];
|
||||
const getModel = vi.spyOn(models, "getModel");
|
||||
const contextKey = createContextKey<string>("accept.context");
|
||||
const context = withContextValue(contextKey, "source", BACKGROUND_CONTEXT);
|
||||
const seen: Array<{ event: HarnessEvent; sameContext: boolean }> = [];
|
||||
for (const type of ["run_start", "message_start", "message_end", "entry_added"] as const) {
|
||||
harness.events.on(type, (event, eventContext) => {
|
||||
seen.push({ event, sameContext: eventContext === context });
|
||||
});
|
||||
}
|
||||
|
||||
const admission = unwrap(
|
||||
await lane.accept({ kind: "prompt", operationId: "operation", prompt: messages }, context),
|
||||
);
|
||||
|
||||
expect(admission).toEqual({ operationId: "operation", kind: "run", startedAt: expect.any(Number) });
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
expect(
|
||||
storage
|
||||
.getCommitAttempts()[0]
|
||||
?.map((write) => (write.kind === "value" ? `${write.kind}:${write.op}` : write.kind)),
|
||||
).toEqual(["entry", "entry", "value:set", "value:set", "value:set", "value:set"]);
|
||||
const operation = lane.state.operation;
|
||||
if (operation?.meta.intent.kind !== "run") throw new Error("Expected run metadata");
|
||||
expect(operation.meta).toMatchObject({
|
||||
operationId: "operation",
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
intent: { promptEntryIds: expect.any(Array) },
|
||||
});
|
||||
expect(operation.meta.intent.promptEntryIds).toHaveLength(2);
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBe(operation.meta.intent.promptEntryIds[1]);
|
||||
expect(seen.map(({ event }) => event.type)).toEqual([
|
||||
"run_start",
|
||||
"message_start",
|
||||
"message_end",
|
||||
"entry_added",
|
||||
"message_start",
|
||||
"message_end",
|
||||
"entry_added",
|
||||
]);
|
||||
expect(seen.every(({ sameContext }) => sameContext)).toBe(true);
|
||||
expect(getModel).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("captures next-run messages before request messages and accepts an otherwise empty request", async () => {
|
||||
const first = "pending-first";
|
||||
const second = "pending-second";
|
||||
const firstMessage = { role: "user" as const, content: "first", timestamp: 1 };
|
||||
const secondMessage = { role: "user" as const, content: "second", timestamp: 2 };
|
||||
const { harness, lane, session, storage } = await createHarness(async (source) => {
|
||||
await source.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.pendingEntry(first), {
|
||||
type: "message",
|
||||
payload: firstMessage,
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingEntry(second), {
|
||||
type: "message",
|
||||
payload: secondMessage,
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [
|
||||
{ entryId: first, kind: "nextRun" },
|
||||
{ entryId: second, kind: "nextRun" },
|
||||
],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
});
|
||||
const events: string[] = [];
|
||||
harness.events.on("queue_update", (event) => {
|
||||
events.push(event.type);
|
||||
});
|
||||
|
||||
unwrap(await lane.accept({ kind: "prompt", prompt: "" }, BACKGROUND_CONTEXT));
|
||||
|
||||
const operation = lane.state.operation;
|
||||
if (operation?.meta.intent.kind !== "run" || operation.state.at !== "starting") {
|
||||
throw new Error("Expected accepted run");
|
||||
}
|
||||
expect(operation.meta.intent.promptEntryIds).toEqual([]);
|
||||
expect(operation.meta.sourceTipId).toBeNull();
|
||||
expect(lane.state.inbox).toEqual([]);
|
||||
expect(
|
||||
(await session.scanBranch({ start: second, order: "oldestFirst" }, BACKGROUND_CONTEXT)).map(({ id }) => id),
|
||||
).toEqual([first, second]);
|
||||
expect(await session.getValue(storedValues.pendingEntry(first), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(await session.getValue(storedValues.pendingEntry(second), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(
|
||||
storage
|
||||
.getCommitAttempts()[0]
|
||||
?.map((write) => (write.kind === "value" ? `${write.kind}:${write.op}` : write.kind)),
|
||||
).toEqual(["entry", "entry", "value:delete", "value:delete", "value:set", "value:set", "value:set", "value:set"]);
|
||||
expect(events).toEqual(["queue_update"]);
|
||||
});
|
||||
|
||||
it("captures every eligible tag by mode and preserves admission order before the request", async () => {
|
||||
const ids = {
|
||||
write: "queued-write",
|
||||
next: "queued-next",
|
||||
steer1: "queued-steer-1",
|
||||
steer2: "queued-steer-2",
|
||||
follow1: "queued-follow-1",
|
||||
follow2: "queued-follow-2",
|
||||
};
|
||||
const { lane, session } = await createHarness(
|
||||
async (source) => {
|
||||
await source.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.pendingEntry(ids.write), {
|
||||
type: "custom",
|
||||
customType: "queued-write",
|
||||
}),
|
||||
...[ids.next, ids.steer1, ids.steer2, ids.follow1, ids.follow2].map((entryId) =>
|
||||
storedValues.setValue(storedValues.pendingEntry(entryId), {
|
||||
type: "message",
|
||||
payload: { role: "user", content: entryId, timestamp: 1 },
|
||||
}),
|
||||
),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [
|
||||
{ entryId: ids.steer1, kind: "steer" },
|
||||
{ entryId: ids.write, kind: "write" },
|
||||
{ entryId: ids.next, kind: "nextRun" },
|
||||
{ entryId: ids.follow1, kind: "followUp" },
|
||||
{ entryId: ids.steer2, kind: "steer" },
|
||||
{ entryId: ids.follow2, kind: "followUp" },
|
||||
],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
},
|
||||
{},
|
||||
{ steeringMode: "one-at-a-time", followUpMode: "one-at-a-time" },
|
||||
);
|
||||
|
||||
unwrap(await lane.accept({ kind: "prompt", prompt: "request" }, BACKGROUND_CONTEXT));
|
||||
|
||||
const tipId = lane.state.tipId;
|
||||
if (tipId === null) throw new Error("Expected accepted tip");
|
||||
const entries = await session.scanBranch({ start: tipId, order: "oldestFirst" }, BACKGROUND_CONTEXT);
|
||||
expect(entries.map((entry) => entry.id)).toEqual([ids.steer1, ids.write, ids.next, ids.follow1, tipId]);
|
||||
expect(lane.state.inbox).toEqual([
|
||||
{ entryId: ids.steer2, kind: "steer" },
|
||||
{ entryId: ids.follow2, kind: "followUp" },
|
||||
]);
|
||||
expect((await session.getValue(storedValues.laneState("main"), BACKGROUND_CONTEXT))?.value.inbox).toEqual(
|
||||
lane.state.inbox,
|
||||
);
|
||||
for (const entryId of [ids.write, ids.next, ids.steer1, ids.follow1]) {
|
||||
expect(await session.getValue(storedValues.pendingEntry(entryId), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
}
|
||||
for (const entryId of [ids.steer2, ids.follow2]) {
|
||||
expect(await session.getValue(storedValues.pendingEntry(entryId), BACKGROUND_CONTEXT)).toBeDefined();
|
||||
}
|
||||
});
|
||||
|
||||
it("does not let a lone queued write validate empty acceptance", async () => {
|
||||
const entryId = "queued-write";
|
||||
const { lane, session } = await createHarness(async (source) => {
|
||||
await source.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.pendingEntry(entryId), {
|
||||
type: "custom",
|
||||
customType: "queued-write",
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [{ entryId, kind: "write" }],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
});
|
||||
|
||||
const result = await lane.accept({ kind: "prompt", prompt: "" }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(result).toMatchObject({ ok: false, error: { _tag: "InvalidMessage", reason: "empty" } });
|
||||
expect(lane.state.inbox).toEqual([{ entryId, kind: "write" }]);
|
||||
expect(await session.getValue(storedValues.pendingEntry(entryId), BACKGROUND_CONTEXT)).toBeDefined();
|
||||
});
|
||||
|
||||
it("formats skills and templates before acceptance", async () => {
|
||||
const resources = {
|
||||
skills: [
|
||||
{ name: "review", description: "Review", content: "Inspect it", filePath: "/skills/review/SKILL.md" },
|
||||
],
|
||||
promptTemplates: [{ name: "fix", content: "Fix $1 then $@" }],
|
||||
};
|
||||
const skill = await createHarness(undefined, resources);
|
||||
unwrap(
|
||||
await skill.lane.accept(
|
||||
{ kind: "skill", name: "review", additionalInstructions: "Be strict" },
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
const skillEntry = await skill.session.getEntry(skill.lane.state.tipId!, BACKGROUND_CONTEXT);
|
||||
expect(skillEntry).toMatchObject({
|
||||
type: "message",
|
||||
message: { content: [{ type: "text", text: expect.stringContaining('<skill name="review"') }] },
|
||||
});
|
||||
|
||||
const template = await createHarness(undefined, resources);
|
||||
unwrap(
|
||||
await template.lane.accept({ kind: "prompt_template", name: "fix", args: ["A", "B"] }, BACKGROUND_CONTEXT),
|
||||
);
|
||||
const templateEntry = await template.session.getEntry(template.lane.state.tipId!, BACKGROUND_CONTEXT);
|
||||
expect(templateEntry).toMatchObject({
|
||||
type: "message",
|
||||
message: { content: [{ type: "text", text: "Fix A then A B" }] },
|
||||
});
|
||||
});
|
||||
|
||||
it("accepts standalone compaction with durable preparation and no execution", async () => {
|
||||
const { harness, lane, session, storage } = await createHarness();
|
||||
await lane.appendMessage({ role: "user", content: "history", timestamp: 1 }, BACKGROUND_CONTEXT);
|
||||
storage.clearCommitAttempts();
|
||||
const starts = vi.fn();
|
||||
harness.events.on("compaction_start", starts);
|
||||
|
||||
const admission = unwrap(
|
||||
await lane.accept(
|
||||
{ kind: "compaction", operationId: "compaction", customInstructions: "focus" },
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
|
||||
expect(admission).toEqual({ operationId: "compaction", kind: "compaction", startedAt: expect.any(Number) });
|
||||
const operation = lane.state.operation;
|
||||
if (operation?.state.at !== "summary.deciding") throw new Error("Expected accepted compaction");
|
||||
expect(operation.state.task).toMatchObject({
|
||||
reason: "manual",
|
||||
customInstructions: "focus",
|
||||
boundary: { kind: "finish" },
|
||||
});
|
||||
expect(
|
||||
await session.getValue(
|
||||
storedValues.operationPreparation("compaction", operation.state.task.taskId),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toMatchObject({ value: { kind: "compaction" } });
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
expect(starts).toHaveBeenCalledTimes(1);
|
||||
expect(lane.activeDrive).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects empty standalone compaction without writing", async () => {
|
||||
const { lane, storage } = await createHarness();
|
||||
|
||||
expect(await lane.accept({ kind: "compaction" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "NothingToCompact" },
|
||||
});
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it.each([false, true])("accepts %s summarized navigation atomically", async (summarize) => {
|
||||
const { harness, lane, session, storage } = await createHarness(async (source) => {
|
||||
await source.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "root",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "root", timestamp: 1 },
|
||||
},
|
||||
},
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "source",
|
||||
parentId: "root",
|
||||
type: "message",
|
||||
message: { role: "user", content: "source", timestamp: 2 },
|
||||
},
|
||||
},
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "target",
|
||||
parentId: "root",
|
||||
type: "message",
|
||||
message: { role: "user", content: "target", timestamp: 3 },
|
||||
},
|
||||
},
|
||||
storedValues.setValue(storedValues.branchTip("main"), "source"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
});
|
||||
storage.clearCommitAttempts();
|
||||
const starts = vi.fn();
|
||||
harness.events.on("navigation_start", starts);
|
||||
|
||||
unwrap(
|
||||
await lane.accept(
|
||||
{
|
||||
kind: "navigation",
|
||||
operationId: summarize ? "summarized" : "direct",
|
||||
targetId: "target",
|
||||
options: { summarize, label: "chosen", customInstructions: "focus" },
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
|
||||
const operation = lane.state.operation;
|
||||
if (operation === null) throw new Error("Expected accepted navigation");
|
||||
expect(operation.state.at).toBe(summarize ? "summary.deciding" : "navigation.ready_to_commit");
|
||||
expect(lane.state.tipId).toBe("source");
|
||||
if (summarize) {
|
||||
if (operation.state.at !== "summary.deciding") throw new Error("Expected summary decision");
|
||||
expect(
|
||||
await session.getValue(
|
||||
storedValues.operationPreparation(operation.meta.operationId, operation.state.task.taskId),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toMatchObject({ value: { kind: "branch_summary", messages: [{ content: "source" }] } });
|
||||
}
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
expect(starts).toHaveBeenCalledTimes(1);
|
||||
expect(lane.activeDrive).toBeUndefined();
|
||||
});
|
||||
|
||||
it("validates navigation before acceptance", async () => {
|
||||
const { lane, storage } = await createHarness(async (source) => {
|
||||
await source.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "source",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "source", timestamp: 1 },
|
||||
},
|
||||
},
|
||||
storedValues.setValue(storedValues.branchTip("main"), "source"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
});
|
||||
|
||||
expect(await lane.accept({ kind: "navigation", targetId: "source" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "InvalidNavigation", reason: "current_tip" },
|
||||
});
|
||||
expect(
|
||||
await lane.accept({ kind: "navigation", targetId: null, options: { label: "bad" } }, BACKGROUND_CONTEXT),
|
||||
).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "InvalidNavigation", reason: "root_label" },
|
||||
});
|
||||
expect(await lane.accept({ kind: "navigation", targetId: "missing" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "UnknownTarget" },
|
||||
});
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it("returns expected pre-acceptance errors without writing", async () => {
|
||||
const { lane, storage } = await createHarness();
|
||||
const pending = fauxAssistantMessage([], { stopReason: "pending" });
|
||||
|
||||
expect(await lane.accept({ kind: "prompt", prompt: "" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "InvalidMessage", reason: "empty" },
|
||||
});
|
||||
expect(await lane.accept({ kind: "prompt", prompt: pending }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "InvalidMessage", reason: "pending_assistant" },
|
||||
});
|
||||
expect(await lane.accept({ kind: "skill", name: "missing" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "UnknownSkill" },
|
||||
});
|
||||
expect(await lane.accept({ kind: "prompt_template", name: "missing" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "UnknownTemplate" },
|
||||
});
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it("serializes concurrent accepts so exactly one wins", async () => {
|
||||
const { lane, storage } = await createHarness();
|
||||
|
||||
const results = await Promise.all([
|
||||
lane.accept({ kind: "prompt", prompt: "first" }, BACKGROUND_CONTEXT),
|
||||
lane.accept({ kind: "prompt", prompt: "second" }, BACKGROUND_CONTEXT),
|
||||
]);
|
||||
|
||||
expect(results.filter((result) => result.ok)).toHaveLength(1);
|
||||
expect(results.filter((result) => !result.ok)).toMatchObject([{ error: { _tag: "LaneBusy" } }]);
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("faults on commit failure without publishing acceptance", async () => {
|
||||
const storage = new FailingMemoryStorage();
|
||||
const session = new StorageBackedSession({ id: "failing", createdAt: 1, storageVersion: 1 }, storage);
|
||||
sessions.push(session);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const { harness } = await createAgentHarness(options(session), BACKGROUND_CONTEXT);
|
||||
if (!(harness instanceof Harness)) throw new Error("Expected runtime Harness");
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
if (!(lane instanceof Lane)) throw new Error("Expected runtime Lane");
|
||||
storage.failure = new Error("accept failed");
|
||||
|
||||
await expect(lane.accept({ kind: "prompt", prompt: "hello" }, BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(
|
||||
HarnessFault,
|
||||
);
|
||||
expect(lane.state.operation).toBeNull();
|
||||
});
|
||||
|
||||
it("delivers acceptance listeners after publishing state and permits serialized reads", async () => {
|
||||
const { harness, lane } = await createHarness();
|
||||
let inspected = false;
|
||||
harness.events.on("run_start", async () => {
|
||||
inspected = (await lane.inspectExecution(BACKGROUND_CONTEXT)).current?.status === "open";
|
||||
});
|
||||
|
||||
unwrap(await lane.accept({ kind: "prompt", prompt: "hello" }, BACKGROUND_CONTEXT));
|
||||
|
||||
expect(inspected).toBe(true);
|
||||
});
|
||||
|
||||
it("does not resolve acceptance before its direct listeners settle", async () => {
|
||||
const { harness, lane } = await createHarness();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
harness.events.on("run_start", async () => {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
});
|
||||
|
||||
const acceptance = lane.accept({ kind: "prompt", prompt: "hello" }, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
let resolved = false;
|
||||
void acceptance.then(() => {
|
||||
resolved = true;
|
||||
});
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(resolved).toBe(false);
|
||||
release.resolve();
|
||||
unwrap(await acceptance);
|
||||
expect(resolved).toBe(true);
|
||||
});
|
||||
|
||||
it("returns Closed when acceptance starts after close", async () => {
|
||||
const { harness, lane, storage } = await createHarness();
|
||||
await harness.close(BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await lane.accept({ kind: "prompt", prompt: "late" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "Closed" },
|
||||
});
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it("publishes an acceptance admitted before close", async () => {
|
||||
const storage = new ControlledMemoryStorage();
|
||||
const session = new StorageBackedSession({ id: "closing", createdAt: 1, storageVersion: 1 }, storage);
|
||||
sessions.push(session);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const { harness } = await createAgentHarness(options(session), BACKGROUND_CONTEXT);
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
storage.beforeNextCommit = async () => {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
};
|
||||
const acceptance = lane.accept({ kind: "prompt", prompt: "hello" }, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
const closing = harness.close(BACKGROUND_CONTEXT);
|
||||
release.resolve();
|
||||
|
||||
expect((await acceptance).ok).toBe(true);
|
||||
await closing;
|
||||
});
|
||||
});
|
||||
790
third_party/pi/packages/agent/test/harness/runtime/drive-generation.test.ts
vendored
Normal file
790
third_party/pi/packages/agent/test/harness/runtime/drive-generation.test.ts
vendored
Normal file
@@ -0,0 +1,790 @@
|
||||
import {
|
||||
type AssistantMessage,
|
||||
createAssistantMessageEventStream,
|
||||
createModels,
|
||||
fauxAssistantMessage,
|
||||
fauxProvider,
|
||||
type MutableModels,
|
||||
type Provider,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { HarnessEvent, WatchHandle } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT, type Context } from "../../../src/harness/context.ts";
|
||||
import { HookRegistry } from "../../../src/harness/hooks.ts";
|
||||
import { runCheckpoint, startRun } from "../../../src/harness/runtime/drive/checkpoint.ts";
|
||||
import { runGeneration } from "../../../src/harness/runtime/drive/generation.ts";
|
||||
import { recoverAssistantGeneration } from "../../../src/harness/runtime/drive/recovery.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { restoreLane } from "../../../src/harness/runtime/restore.ts";
|
||||
import { type Config, Drive } from "../../../src/harness/runtime/types.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import { InstrumentedStorage } from "../../../src/harness/session/testing/instrumented-storage.ts";
|
||||
import {
|
||||
type AssistantEffectPendingOperation,
|
||||
type AssistantReadyOperation,
|
||||
type CheckpointOperation,
|
||||
type LaneConfiguration,
|
||||
type MessageEntry,
|
||||
type OperationState,
|
||||
operationScopeOf,
|
||||
type Session,
|
||||
type Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import { deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const operationId = "01950000-0000-7000-8000-000000000001";
|
||||
|
||||
class FrameBlockingMemoryStorage extends MemoryStorage {
|
||||
blockNextFrame = false;
|
||||
readonly frameStarted = deferred();
|
||||
readonly releaseFrame = deferred();
|
||||
|
||||
override async commit(writes: Write[], context: Context) {
|
||||
if (
|
||||
this.blockNextFrame &&
|
||||
writes.some(
|
||||
(write) =>
|
||||
write.kind === "list" && write.op === "append" && write.namespace === "pi.pending.assistant_frame",
|
||||
)
|
||||
) {
|
||||
this.blockNextFrame = false;
|
||||
this.frameStarted.resolve();
|
||||
await this.releaseFrame.promise;
|
||||
}
|
||||
return super.commit(writes, context);
|
||||
}
|
||||
}
|
||||
|
||||
interface Fixture {
|
||||
lane: Lane<undefined>;
|
||||
drive: Drive;
|
||||
session: Session;
|
||||
storage: InstrumentedStorage;
|
||||
models: MutableModels;
|
||||
faux: ReturnType<typeof fauxProvider>;
|
||||
hooks: HookRegistry;
|
||||
events: HarnessEvent[];
|
||||
config: Config<undefined>;
|
||||
}
|
||||
|
||||
function unusedWatch<T>(): WatchHandle<T> {
|
||||
throw new Error("watch is not used by generation tests");
|
||||
}
|
||||
|
||||
async function createFixture(backend: MemoryStorage = new MemoryStorage({ now: () => 100 })): Promise<Fixture> {
|
||||
const storage = new InstrumentedStorage(backend);
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `generation-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
const faux = fauxProvider({ tokenSize: { min: 1, max: 1 } });
|
||||
const model = faux.getModel();
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: model.provider, modelId: model.id },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
};
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
const hooks = new HookRegistry(() => {});
|
||||
const events: HarnessEvent[] = [];
|
||||
const config: Config<undefined> = {
|
||||
tools: [],
|
||||
resources: {},
|
||||
streamOptions: {},
|
||||
retryPolicy: { enabled: true, maxRetries: 3, baseDelayMs: 1 },
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
toolContext: undefined,
|
||||
systemPrompt: undefined,
|
||||
toProviderMessages: (messages) =>
|
||||
messages.filter(
|
||||
(message) => message.role === "user" || message.role === "assistant" || message.role === "toolResult",
|
||||
),
|
||||
entryProjectors: {},
|
||||
};
|
||||
const lane = new Lane<undefined>(
|
||||
"main",
|
||||
session,
|
||||
models,
|
||||
hooks,
|
||||
await restoreLane(session, "main", BACKGROUND_CONTEXT),
|
||||
(cause) => (cause instanceof Error ? cause : new Error(String(cause))),
|
||||
(batch) => {
|
||||
events.push(...structuredClone(batch));
|
||||
return Promise.resolve();
|
||||
},
|
||||
unusedWatch,
|
||||
() => config,
|
||||
);
|
||||
const admission = await lane.accept({ kind: "prompt", operationId, prompt: "question" }, BACKGROUND_CONTEXT);
|
||||
if (!admission.ok) throw admission.error;
|
||||
const drive = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
lane.activeDrive = drive;
|
||||
storage.clearCommitAttempts();
|
||||
return { lane, drive, session, storage, models, faux, hooks, events, config };
|
||||
}
|
||||
|
||||
function currentRun(lane: Lane<undefined>): OperationState {
|
||||
const operation = lane.state.operation;
|
||||
if (operation === null) throw new Error("fixture has no operation");
|
||||
return operation.state;
|
||||
}
|
||||
|
||||
function readyGeneration(lane: Lane<undefined>): AssistantReadyOperation {
|
||||
const run = currentRun(lane);
|
||||
if (run.at !== "assistant.ready") throw new Error("fixture has no ready generation");
|
||||
return run;
|
||||
}
|
||||
|
||||
async function startFixtureRun(fixture: Fixture) {
|
||||
const run = currentRun(fixture.lane);
|
||||
if (run.at !== "starting") throw new Error("run did not start at its initial boundary");
|
||||
return startRun(fixture.lane, fixture.drive, run);
|
||||
}
|
||||
|
||||
async function advanceToReady(fixture: Fixture): Promise<AssistantReadyOperation> {
|
||||
await startFixtureRun(fixture);
|
||||
const run = currentRun(fixture.lane);
|
||||
if (run.at !== "checkpoint") throw new Error("run did not reach checkpoint");
|
||||
await runCheckpoint(fixture.lane, fixture.drive, run);
|
||||
return readyGeneration(fixture.lane);
|
||||
}
|
||||
|
||||
async function expectProjectionRestores(fixture: Fixture): Promise<void> {
|
||||
expect(fixture.lane.state).toEqual(await restoreLane(fixture.session, "main", BACKGROUND_CONTEXT));
|
||||
}
|
||||
|
||||
function writesOperationState(writes: readonly Write[], status: string): boolean {
|
||||
return writes.some(
|
||||
(write) =>
|
||||
write.kind === "value" &&
|
||||
write.op === "set" &&
|
||||
write.namespace === "pi.op.state" &&
|
||||
typeof write.value === "object" &&
|
||||
write.value !== null &&
|
||||
"at" in write.value &&
|
||||
write.value.at === `assistant.${status}`,
|
||||
);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime generation checkpoint", () => {
|
||||
it("consumes before_run and snapshots a ready generation", async () => {
|
||||
const fixture = await createFixture();
|
||||
fixture.hooks.on("before_run", () => ({
|
||||
messages: [{ role: "user", content: "injected", timestamp: 2 }],
|
||||
}));
|
||||
expect(await startFixtureRun(fixture)).toEqual({ kind: "continue" });
|
||||
let run = currentRun(fixture.lane);
|
||||
if (run.at !== "checkpoint") throw new Error("missing checkpoint");
|
||||
expect(run.triggerEntryId).toBe(fixture.lane.state.tipId);
|
||||
expect(fixture.events.map((event) => event.type)).toContain("entry_added");
|
||||
await expectProjectionRestores(fixture);
|
||||
|
||||
expect(await runCheckpoint(fixture.lane, fixture.drive, run)).toEqual({ kind: "continue" });
|
||||
run = currentRun(fixture.lane);
|
||||
if (run.at !== "assistant.ready") {
|
||||
throw new Error("missing ready generation");
|
||||
}
|
||||
expect(run.generationContext).toMatchObject({
|
||||
configuration: fixture.lane.state.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 4, baseDelayMs: 1 },
|
||||
overflowRecoveryUsed: false,
|
||||
});
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("drains mixed pending writes while separating leaf and generation trigger", async () => {
|
||||
const fixture = await createFixture();
|
||||
await startFixtureRun(fixture);
|
||||
const messageId = "01950000-0000-7000-8000-000000000010";
|
||||
const customId = "01950000-0000-7000-8000-000000000011";
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const inbox = [
|
||||
...state.inbox,
|
||||
{ entryId: messageId, kind: "write" as const },
|
||||
{ entryId: customId, kind: "write" as const },
|
||||
];
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [
|
||||
storedValues.setValue(storedValues.pendingEntry(messageId), {
|
||||
type: "message",
|
||||
payload: { role: "user", content: "projecting", timestamp: 3 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingEntry(customId), {
|
||||
type: "custom",
|
||||
customType: "display-only",
|
||||
payload: { value: true },
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox,
|
||||
}),
|
||||
],
|
||||
next: { ...state, inbox },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
const run = currentRun(fixture.lane);
|
||||
if (run.at !== "checkpoint") throw new Error("missing checkpoint");
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
await runCheckpoint(fixture.lane, fixture.drive, run);
|
||||
|
||||
expect(fixture.storage.getCommitAttempts()).toHaveLength(1);
|
||||
const routed = currentRun(fixture.lane);
|
||||
if (routed.at !== "assistant.ready") throw new Error("pending writes did not route to generation");
|
||||
expect(fixture.lane.state.tipId).toBe(customId);
|
||||
expect(routed.generationContext).toMatchObject({
|
||||
triggerEntryId: messageId,
|
||||
overflowRecoveryUsed: false,
|
||||
});
|
||||
expect(fixture.lane.state.inbox).toEqual([]);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("preserves checkpoint routing when a custom-only pending write does not project", async () => {
|
||||
const fixture = await createFixture();
|
||||
await startFixtureRun(fixture);
|
||||
const current = currentRun(fixture.lane);
|
||||
if (current.at !== "checkpoint") throw new Error("missing checkpoint");
|
||||
const customId = "01950000-0000-7000-8000-000000000012";
|
||||
const checkpoint: CheckpointOperation = {
|
||||
...current,
|
||||
continuation: { kind: "may_finish", includeFinalAssistant: false },
|
||||
};
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const inbox = [...state.inbox, { entryId: customId, kind: "write" as const }];
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [
|
||||
storedValues.setValue(storedValues.pendingEntry(customId), {
|
||||
type: "custom",
|
||||
customType: "display-only",
|
||||
payload: { value: true },
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationState(operationId), checkpoint),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox,
|
||||
}),
|
||||
],
|
||||
next: { ...state, inbox, operation: { meta: operation.meta, state: checkpoint } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
fixture.hooks.on("before_run_end", () => ({ followUp: "continue after custom write" }));
|
||||
|
||||
expect(await runCheckpoint(fixture.lane, fixture.drive, checkpoint)).toEqual({ kind: "continue" });
|
||||
|
||||
const ready = currentRun(fixture.lane);
|
||||
if (ready.at !== "assistant.ready") throw new Error("hook follow-up did not route to generation");
|
||||
const followUpId = ready.generationContext.triggerEntryId;
|
||||
expect(followUpId).toBe(fixture.lane.state.tipId);
|
||||
expect(await fixture.session.getEntry(customId, BACKGROUND_CONTEXT)).toMatchObject({ type: "custom" });
|
||||
expect(await fixture.session.getEntry(followUpId, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
parentId: customId,
|
||||
type: "message",
|
||||
message: { role: "user", content: "continue after custom write" },
|
||||
});
|
||||
expect(fixture.lane.state.inbox).toEqual([]);
|
||||
expect(await fixture.session.getValue(storedValues.pendingEntry(customId), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("drops a stale finish-hook follow-up when input arrives during the hook", async () => {
|
||||
const fixture = await createFixture();
|
||||
await startFixtureRun(fixture);
|
||||
const current = currentRun(fixture.lane);
|
||||
if (current.at !== "checkpoint") throw new Error("missing checkpoint");
|
||||
const checkpoint: CheckpointOperation = {
|
||||
...current,
|
||||
continuation: { kind: "may_finish", includeFinalAssistant: false },
|
||||
};
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(operationId), checkpoint)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: checkpoint } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
const hookStarted = deferred();
|
||||
const releaseHook = deferred();
|
||||
fixture.hooks.on("before_run_end", async () => {
|
||||
hookStarted.resolve();
|
||||
await releaseHook.promise;
|
||||
return { followUp: "stale hook follow-up" };
|
||||
});
|
||||
|
||||
const running = runCheckpoint(fixture.lane, fixture.drive, checkpoint);
|
||||
await hookStarted.promise;
|
||||
await fixture.lane.command((state) => {
|
||||
const inbox = [...state.inbox, { entryId: "steer-during-finish", kind: "steer" as const }];
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [
|
||||
storedValues.setValue(storedValues.pendingEntry("steer-during-finish"), {
|
||||
type: "message",
|
||||
payload: { role: "user", content: "new input", timestamp: 4 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox,
|
||||
}),
|
||||
],
|
||||
next: { ...state, inbox },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
releaseHook.resolve();
|
||||
|
||||
expect(await running).toEqual({ kind: "continue" });
|
||||
const ready = currentRun(fixture.lane);
|
||||
if (ready.at !== "assistant.ready") throw new Error("new input did not replace stale finish decision");
|
||||
expect(ready.generationContext.triggerEntryId).toBe("steer-during-finish");
|
||||
expect(fixture.lane.state.tipId).toBe("steer-during-finish");
|
||||
expect((await fixture.session.getEntry("steer-during-finish", BACKGROUND_CONTEXT))?.type).toBe("message");
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime assistant generation", () => {
|
||||
it("commits intent before provider admission, preserves queued inbox state, and settles reserved ids", async () => {
|
||||
const fixture = await createFixture();
|
||||
const ready = await advanceToReady(fixture);
|
||||
const hookStarted = deferred();
|
||||
const releaseHook = deferred();
|
||||
fixture.hooks.on("before_request", async () => {
|
||||
hookStarted.resolve();
|
||||
await releaseHook.promise;
|
||||
return undefined;
|
||||
});
|
||||
let effectPendingAtProvider = false;
|
||||
fixture.faux.setResponses([
|
||||
async () => {
|
||||
const state = await fixture.session.getValue(storedValues.operationState(operationId), BACKGROUND_CONTEXT);
|
||||
effectPendingAtProvider = state?.value.at === "assistant.effect_pending";
|
||||
return fauxAssistantMessage("answer", { timestamp: 5 });
|
||||
},
|
||||
]);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
const generating = runGeneration(fixture.lane, fixture.drive, ready);
|
||||
await hookStarted.promise;
|
||||
const steerId = "01950000-0000-7000-8000-000000000020";
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const inbox = [...state.inbox, { entryId: steerId, kind: "steer" as const }];
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [
|
||||
storedValues.setValue(storedValues.pendingEntry(steerId), {
|
||||
type: "message",
|
||||
payload: { role: "user", content: "late steer", timestamp: 4 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox,
|
||||
}),
|
||||
],
|
||||
next: { ...state, inbox },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
releaseHook.resolve();
|
||||
|
||||
expect(await generating).toEqual({ kind: "continue" });
|
||||
expect(effectPendingAtProvider).toBe(true);
|
||||
const settled = currentRun(fixture.lane);
|
||||
expect(fixture.lane.state.inbox).toEqual([{ entryId: steerId, kind: "steer" }]);
|
||||
if (settled.at !== "checkpoint") throw new Error("response did not reach checkpoint");
|
||||
const responseId = settled.latestAssistantEntryId;
|
||||
if (responseId === null) throw new Error("missing response id");
|
||||
const entry = await fixture.session.getEntry(responseId, BACKGROUND_CONTEXT);
|
||||
expect(entry).toMatchObject({ id: responseId, type: "message", message: { content: [{ text: "answer" }] } });
|
||||
expect(
|
||||
await fixture.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, responseId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
const attempts = fixture.storage.getCommitAttempts();
|
||||
expect(attempts.some((writes) => writesOperationState(writes, "effect_pending"))).toBe(true);
|
||||
const settlement = attempts.find((writes) => writes.some((write) => write.kind === "entry"));
|
||||
expect(
|
||||
settlement?.map((write) =>
|
||||
write.kind === "value" || write.kind === "list"
|
||||
? `${write.kind}:${write.op}:${write.namespace}`
|
||||
: write.kind,
|
||||
),
|
||||
).toEqual([
|
||||
"entry",
|
||||
"usage",
|
||||
"value:set:pi.branch.tip",
|
||||
"list:delete:pi.pending.assistant_frame",
|
||||
"value:set:pi.op.state",
|
||||
]);
|
||||
expect(fixture.events.map((event) => event.type)).toEqual(
|
||||
expect.arrayContaining([
|
||||
"turn_start",
|
||||
"message_start",
|
||||
"message_update",
|
||||
"message_end",
|
||||
"entry_added",
|
||||
"usage",
|
||||
"turn_end",
|
||||
]),
|
||||
);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("does not await frame storage inside the provider event loop and persists frames in order", async () => {
|
||||
const backend = new FrameBlockingMemoryStorage({ now: () => 100 });
|
||||
const fixture = await createFixture(backend);
|
||||
const ready = await advanceToReady(fixture);
|
||||
const base = fixture.faux.provider;
|
||||
const provider: Provider = {
|
||||
id: base.id,
|
||||
name: base.name,
|
||||
auth: base.auth,
|
||||
getModels: () => base.getModels(),
|
||||
stream: (model, context, options) => base.stream(model, context, options),
|
||||
streamSimple: () => {
|
||||
const stream = createAssistantMessageEventStream();
|
||||
const partial: AssistantMessage = {
|
||||
...fauxAssistantMessage([], { timestamp: 5 }),
|
||||
stopReason: "pending",
|
||||
};
|
||||
const text = { type: "text" as const, text: "" };
|
||||
queueMicrotask(() => {
|
||||
stream.push({ type: "start", partial });
|
||||
partial.content.push(text);
|
||||
stream.push({ type: "text_start", contentIndex: 0, partial });
|
||||
text.text = "ab";
|
||||
stream.push({ type: "text_delta", contentIndex: 0, delta: "ab", partial });
|
||||
stream.push({ type: "text_end", contentIndex: 0, content: "ab", partial });
|
||||
stream.push({ type: "done", reason: "stop", message: fauxAssistantMessage("ab", { timestamp: 5 }) });
|
||||
});
|
||||
return stream;
|
||||
},
|
||||
};
|
||||
fixture.models.setProvider(provider);
|
||||
backend.blockNextFrame = true;
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
const generating = runGeneration(fixture.lane, fixture.drive, ready);
|
||||
await backend.frameStarted.promise;
|
||||
await expect
|
||||
.poll(() => fixture.events.filter((event) => event.type === "message_update").length)
|
||||
.toBeGreaterThan(1);
|
||||
backend.releaseFrame.resolve();
|
||||
await generating;
|
||||
|
||||
const frames = fixture.storage
|
||||
.getCommitAttempts()
|
||||
.flatMap((writes) =>
|
||||
writes.flatMap((write) =>
|
||||
write.kind === "list" && write.op === "append" && write.namespace === "pi.pending.assistant_frame"
|
||||
? [write.value]
|
||||
: [],
|
||||
),
|
||||
);
|
||||
expect(
|
||||
frames.map((frame) =>
|
||||
typeof frame === "object" && frame !== null && "type" in frame ? frame.type : undefined,
|
||||
),
|
||||
).toEqual(["start", "text_start", "text_end"]);
|
||||
expect(frames[0]).toMatchObject({ partial: { content: [] } });
|
||||
expect(frames[1]).toMatchObject({ content: { text: "ab" } });
|
||||
});
|
||||
|
||||
it("enters configuration failure without reserving response ids or calling the provider", async () => {
|
||||
const fixture = await createFixture();
|
||||
await fixture.lane.setActiveTools(["missing"], BACKGROUND_CONTEXT);
|
||||
const ready = await advanceToReady(fixture);
|
||||
const queuedId = await fixture.lane.appendCustomEntry("after-failure", { retained: true }, BACKGROUND_CONTEXT);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
expect(await runGeneration(fixture.lane, fixture.drive, ready)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: {
|
||||
operationId,
|
||||
kind: "run",
|
||||
status: "failed",
|
||||
error: { code: "configured_tools_unavailable", details: { tools: ["missing"] } },
|
||||
},
|
||||
});
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
expect(fixture.lane.state.inbox).toEqual([{ entryId: queuedId, kind: "write" }]);
|
||||
expect(await fixture.session.getValue(storedValues.pendingEntry(queuedId), BACKGROUND_CONTEXT)).toBeDefined();
|
||||
expect(fixture.faux.state.callCount).toBe(0);
|
||||
expect(
|
||||
fixture.storage
|
||||
.getCommitAttempts()
|
||||
.flat()
|
||||
.some((write) => write.kind === "entry" || write.kind === "usage"),
|
||||
).toBe(false);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("fails a missing captured model before reserving ids", async () => {
|
||||
const fixture = await createFixture();
|
||||
const ready = await advanceToReady(fixture);
|
||||
fixture.models.deleteProvider(fixture.faux.provider.id);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
expect(await runGeneration(fixture.lane, fixture.drive, ready)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, kind: "run", status: "failed", error: { code: "model_unavailable" } },
|
||||
});
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
expect(fixture.faux.state.callCount).toBe(0);
|
||||
expect(
|
||||
fixture.storage
|
||||
.getCommitAttempts()
|
||||
.flat()
|
||||
.some((write) => write.kind === "entry" || write.kind === "usage"),
|
||||
).toBe(false);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("declines intent when cancellation wins preparation", async () => {
|
||||
const cancelled = await createFixture();
|
||||
const cancelledReady = await advanceToReady(cancelled);
|
||||
const cancelHookStarted = deferred();
|
||||
const releaseCancelHook = deferred();
|
||||
cancelled.hooks.on("before_request", async () => {
|
||||
cancelHookStarted.resolve();
|
||||
await releaseCancelHook.promise;
|
||||
return undefined;
|
||||
});
|
||||
cancelled.storage.clearCommitAttempts();
|
||||
const cancelling = runGeneration(cancelled.lane, cancelled.drive, cancelledReady);
|
||||
await cancelHookStarted.promise;
|
||||
await cancelled.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const nextState: OperationState = {
|
||||
...operation.state,
|
||||
control: { status: "cancel_requested", requestedAt: 10 },
|
||||
};
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(operationId), nextState)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: nextState } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
releaseCancelHook.resolve();
|
||||
expect(await cancelling).toEqual({ kind: "continue" });
|
||||
expect(cancelled.faux.state.callCount).toBe(0);
|
||||
expect(
|
||||
cancelled.storage.getCommitAttempts().some((writes) => writesOperationState(writes, "effect_pending")),
|
||||
).toBe(false);
|
||||
});
|
||||
|
||||
it("recovers an orphan with no frames into a zero-usage retry wait", async () => {
|
||||
const fixture = await createFixture();
|
||||
const ready = await advanceToReady(fixture);
|
||||
const responseEntryId = "01950000-0000-7000-8000-000000000028";
|
||||
const usageId = "01950000-0000-7000-8000-000000000029";
|
||||
const pending: AssistantEffectPendingOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: { ...ready.generationContext, retryPolicy: { maxAttempts: 2, baseDelayMs: 1 } },
|
||||
attempt: 1,
|
||||
responseEntryId,
|
||||
usageId,
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1_000,
|
||||
};
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(operationId), pending)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: pending } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
await recoverAssistantGeneration(fixture.lane, fixture.drive, pending);
|
||||
|
||||
const entry = (await fixture.session.getEntry(responseEntryId, BACKGROUND_CONTEXT)) as MessageEntry;
|
||||
expect(entry.message).toMatchObject({
|
||||
api: "unknown",
|
||||
provider: ready.generationContext.configuration.model.provider,
|
||||
model: ready.generationContext.configuration.model.modelId,
|
||||
content: [],
|
||||
stopReason: "error",
|
||||
usage: { totalTokens: 0 },
|
||||
});
|
||||
expect(currentRun(fixture.lane)).toMatchObject({
|
||||
at: "assistant.retry_wait",
|
||||
nextAttempt: 2,
|
||||
});
|
||||
expect(fixture.faux.state.callCount).toBe(0);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("recovers an orphan from committed frames without a provider or response hook", async () => {
|
||||
const fixture = await createFixture();
|
||||
const ready = await advanceToReady(fixture);
|
||||
const responseEntryId = "01950000-0000-7000-8000-000000000030";
|
||||
const usageId = "01950000-0000-7000-8000-000000000031";
|
||||
const partial: AssistantMessage = {
|
||||
...fauxAssistantMessage([], { timestamp: 6 }),
|
||||
stopReason: "pending",
|
||||
};
|
||||
const pending: AssistantEffectPendingOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: { ...ready.generationContext, retryPolicy: { maxAttempts: 1, baseDelayMs: 1 } },
|
||||
attempt: 1,
|
||||
responseEntryId,
|
||||
usageId,
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1_000,
|
||||
};
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [
|
||||
storedValues.setValue(storedValues.operationState(operationId), pending),
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, responseEntryId), {
|
||||
type: "start",
|
||||
partial,
|
||||
}),
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, responseEntryId), {
|
||||
type: "text_start",
|
||||
contentIndex: 0,
|
||||
content: { type: "text", text: "" },
|
||||
}),
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, responseEntryId), {
|
||||
type: "text_delta",
|
||||
contentIndex: 0,
|
||||
delta: "draft",
|
||||
}),
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, responseEntryId), {
|
||||
type: "text_end",
|
||||
contentIndex: 0,
|
||||
content: "corrected",
|
||||
}),
|
||||
],
|
||||
next: { ...state, operation: { meta: operation.meta, state: pending } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
let afterResponseCalls = 0;
|
||||
fixture.hooks.on("after_response", () => {
|
||||
afterResponseCalls++;
|
||||
return undefined;
|
||||
});
|
||||
|
||||
expect(await recoverAssistantGeneration(fixture.lane, fixture.drive, pending)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, kind: "run", status: "failed", tipId: responseEntryId },
|
||||
});
|
||||
|
||||
expect(fixture.faux.state.callCount).toBe(0);
|
||||
expect(afterResponseCalls).toBe(0);
|
||||
const entry = (await fixture.session.getEntry(responseEntryId, BACKGROUND_CONTEXT)) as MessageEntry;
|
||||
expect(entry.message).toMatchObject({
|
||||
role: "assistant",
|
||||
content: [{ type: "text", text: "corrected" }],
|
||||
stopReason: "error",
|
||||
usage: { input: 0, output: 0 },
|
||||
});
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
expect(
|
||||
await fixture.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
fixture.events.filter((event) => "recovery" in event && event.recovery).map((event) => event.type),
|
||||
).toEqual(["message_start", "message_end", "entry_added"]);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("finishes the no-tool run with terminal cleanup and an immutable result record", async () => {
|
||||
const fixture = await createFixture();
|
||||
const ready = await advanceToReady(fixture);
|
||||
fixture.faux.setResponses([fauxAssistantMessage("done", { timestamp: 7 })]);
|
||||
await runGeneration(fixture.lane, fixture.drive, ready);
|
||||
const run = currentRun(fixture.lane);
|
||||
if (run.at !== "checkpoint") throw new Error("missing terminal checkpoint");
|
||||
|
||||
const result = await runCheckpoint(fixture.lane, fixture.drive, run);
|
||||
|
||||
expect(result).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, kind: "run", status: "completed", tipId: fixture.lane.state.tipId },
|
||||
});
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
expect(fixture.lane.state.lastOperationId).toBe(operationId);
|
||||
const getEntries = vi.spyOn(fixture.storage, "getEntries");
|
||||
expect(await fixture.lane.getResult(operationId, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
operationId,
|
||||
kind: "run",
|
||||
status: "completed",
|
||||
tipId: fixture.lane.state.tipId,
|
||||
});
|
||||
expect(await fixture.lane.getResult("unknown", BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(getEntries).not.toHaveBeenCalled();
|
||||
expect(
|
||||
await fixture.session.getValue(storedValues.operationMeta(operationId), BACKGROUND_CONTEXT),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
await fixture.session.getValue(storedValues.operationState(operationId), BACKGROUND_CONTEXT),
|
||||
).toBeUndefined();
|
||||
expect(fixture.events.filter((event) => event.type === "run_end")).toHaveLength(1);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
});
|
||||
612
third_party/pi/packages/agent/test/harness/runtime/drive-public.test.ts
vendored
Normal file
612
third_party/pi/packages/agent/test/harness/runtime/drive-public.test.ts
vendored
Normal file
@@ -0,0 +1,612 @@
|
||||
import { createModels, fauxAssistantMessage, fauxProvider } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import {
|
||||
AgentHarness,
|
||||
type AgentLane,
|
||||
HarnessClosed,
|
||||
HarnessFault,
|
||||
type OperationAdmission,
|
||||
type Resources,
|
||||
} from "../../../src/harness/agent-harness.ts";
|
||||
import { BACKGROUND_CONTEXT, withCancel } from "../../../src/harness/context.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type { Session } from "../../../src/harness/session/types.ts";
|
||||
import { deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
|
||||
async function createFixture(options: { deferred?: boolean; resources?: Resources } = {}): Promise<{
|
||||
lane: AgentLane;
|
||||
runtimeLane: Lane<object | undefined>;
|
||||
harness: Awaited<ReturnType<typeof AgentHarness.create>>["harness"];
|
||||
faux: ReturnType<typeof fauxProvider>;
|
||||
session: Session;
|
||||
}> {
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `public-drive-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
new MemoryStorage(),
|
||||
);
|
||||
sessions.push(session);
|
||||
const faux = fauxProvider();
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
const { harness } = await AgentHarness.create(
|
||||
{
|
||||
session,
|
||||
models,
|
||||
model: faux.getModel(),
|
||||
...(options.deferred === true ? { streamOptions: { deferred: true } } : {}),
|
||||
...(options.resources === undefined ? {} : { resources: options.resources }),
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
if (!(lane instanceof Lane)) throw new Error("Expected runtime Lane");
|
||||
return { lane, runtimeLane: lane, harness, faux, session };
|
||||
}
|
||||
|
||||
async function acceptRun(lane: AgentLane, operationId: string): Promise<OperationAdmission> {
|
||||
const result = await lane.accept({ kind: "prompt", operationId, prompt: operationId }, BACKGROUND_CONTEXT);
|
||||
if (!result.ok) throw result.error;
|
||||
return result.value;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime public drive", () => {
|
||||
it("persists model identity without requiring a local registration", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
await lane.setModel({ provider: "missing", modelId: "missing-model" }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await lane.getModel(BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(await lane.prompt("prompt", undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
kind: "run",
|
||||
status: "failed",
|
||||
error: { code: "model_unavailable" },
|
||||
},
|
||||
});
|
||||
expect(faux.state.callCount).toBe(0);
|
||||
});
|
||||
|
||||
it("composes prompt, skill, and template acceptance with drive", async () => {
|
||||
const resources: Resources = {
|
||||
skills: [
|
||||
{ name: "review", description: "Review", content: "Inspect it", filePath: "/skills/review/SKILL.md" },
|
||||
],
|
||||
promptTemplates: [{ name: "fix", content: "Fix $1" }],
|
||||
};
|
||||
const { lane, faux } = await createFixture({ resources });
|
||||
faux.setResponses([
|
||||
fauxAssistantMessage("prompt answer"),
|
||||
fauxAssistantMessage("skill answer"),
|
||||
fauxAssistantMessage("template answer"),
|
||||
]);
|
||||
|
||||
expect(await lane.prompt("prompt", undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "run", status: "completed" },
|
||||
});
|
||||
expect(await lane.skill("review", "strict", BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "run", status: "completed" },
|
||||
});
|
||||
expect(await lane.promptFromTemplate("fix", ["it"], BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "run", status: "completed" },
|
||||
});
|
||||
expect(faux.state.callCount).toBe(3);
|
||||
});
|
||||
|
||||
it("returns a convenience-only suspension observation", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture({ deferred: true });
|
||||
faux.setResponses([fauxAssistantMessage("eventual answer")]);
|
||||
|
||||
expect(await lane.prompt("defer", undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
operationId: expect.any(String),
|
||||
status: "suspended",
|
||||
deferred: { provider: "faux", modelId: "faux-1" },
|
||||
},
|
||||
});
|
||||
expect(runtimeLane.state.operation?.state.at).toBe("deferred.suspended");
|
||||
});
|
||||
|
||||
it("records caller usage as an adjustment and publishes committed totals", async () => {
|
||||
const { lane, harness, session } = await createFixture();
|
||||
const usage = fauxAssistantMessage("usage").usage;
|
||||
const events: unknown[] = [];
|
||||
harness.events.on("usage", (event) => {
|
||||
events.push(event);
|
||||
});
|
||||
|
||||
const recorded = await lane.recordUsage(
|
||||
usage,
|
||||
{ entryId: "external", details: { source: "test" } },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
if (!recorded.ok) throw recorded.error;
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
type: "usage",
|
||||
lane: "main",
|
||||
row: expect.objectContaining({ id: recorded.value.usageId, adjustment: true }),
|
||||
totals: (await session.getStats(BACKGROUND_CONTEXT)).usage,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it.each(["steer", "followUp", "nextRun"] as const)(
|
||||
"starts an ordinary continuation run from queued %s input",
|
||||
async (kind) => {
|
||||
const { lane, faux } = await createFixture();
|
||||
await lane.appendMessage({ role: "user", content: "history", timestamp: 1 }, BACKGROUND_CONTEXT);
|
||||
const summaryStarted = deferred();
|
||||
const releaseSummary = deferred();
|
||||
faux.setResponses([
|
||||
async () => {
|
||||
summaryStarted.resolve();
|
||||
await releaseSummary.promise;
|
||||
return fauxAssistantMessage("summary");
|
||||
},
|
||||
fauxAssistantMessage("continuation answer"),
|
||||
]);
|
||||
const compacting = lane.compact(undefined, BACKGROUND_CONTEXT);
|
||||
await summaryStarted.promise;
|
||||
const queued = await lane[kind]("continue", undefined, BACKGROUND_CONTEXT);
|
||||
expect(queued.ok).toBe(true);
|
||||
releaseSummary.resolve();
|
||||
|
||||
expect(await compacting).toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
compaction: { kind: "compaction", status: "completed" },
|
||||
run: { kind: "run", status: "completed" },
|
||||
},
|
||||
});
|
||||
expect(faux.state.callCount).toBe(2);
|
||||
},
|
||||
);
|
||||
|
||||
it("lets a competing acceptance win the structural continuation window", async () => {
|
||||
const { lane, harness, faux } = await createFixture();
|
||||
await lane.appendMessage({ role: "user", content: "history", timestamp: 1 }, BACKGROUND_CONTEXT);
|
||||
const summaryStarted = deferred();
|
||||
const releaseSummary = deferred();
|
||||
faux.setResponses([
|
||||
async () => {
|
||||
summaryStarted.resolve();
|
||||
await releaseSummary.promise;
|
||||
return fauxAssistantMessage("summary");
|
||||
},
|
||||
fauxAssistantMessage("competitor answer"),
|
||||
]);
|
||||
let competing: ReturnType<typeof lane.accept> | undefined;
|
||||
harness.events.on("compaction_end", (event) => {
|
||||
if (event.status === "completed") {
|
||||
competing = lane.accept({ kind: "prompt", prompt: "competitor" }, BACKGROUND_CONTEXT);
|
||||
}
|
||||
});
|
||||
const compacting = lane.compact(undefined, BACKGROUND_CONTEXT);
|
||||
await summaryStarted.promise;
|
||||
await lane.nextRun("queued", undefined, BACKGROUND_CONTEXT);
|
||||
releaseSummary.resolve();
|
||||
|
||||
const compacted = await compacting;
|
||||
expect(compacted).toMatchObject({ ok: true, value: { compaction: { status: "completed" } } });
|
||||
if (!compacted.ok) throw compacted.error;
|
||||
expect(compacted.value).not.toHaveProperty("run");
|
||||
if (competing === undefined) throw new Error("Competing acceptance did not start");
|
||||
const admission = await competing;
|
||||
if (!admission.ok) throw admission.error;
|
||||
expect(await lane.drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { status: "completed" } },
|
||||
});
|
||||
});
|
||||
|
||||
it("cancels queued input and reports consumed or missing ids", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
const cancelled = await lane.nextRun("cancel", undefined, BACKGROUND_CONTEXT);
|
||||
if (!cancelled.ok) throw cancelled.error;
|
||||
expect(await lane.cancelQueued(cancelled.value.entryId, BACKGROUND_CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
value: { kind: "cancelled" },
|
||||
});
|
||||
expect(await lane.cancelQueued(cancelled.value.entryId, BACKGROUND_CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
value: { kind: "not_found" },
|
||||
});
|
||||
|
||||
const consumed = await lane.nextRun("consume", undefined, BACKGROUND_CONTEXT);
|
||||
if (!consumed.ok) throw consumed.error;
|
||||
const admission = await lane.accept({ kind: "prompt", prompt: "" }, BACKGROUND_CONTEXT);
|
||||
if (!admission.ok) throw admission.error;
|
||||
expect(await lane.cancelQueued(consumed.value.entryId, BACKGROUND_CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
value: { kind: "already_consumed" },
|
||||
});
|
||||
faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
await lane.drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("admits input after cancellation and preserves it through reconciliation", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
await acceptRun(lane, "cancelled");
|
||||
await lane.requestAbort("cancelled", BACKGROUND_CONTEXT);
|
||||
const late = await lane.steer("late", undefined, BACKGROUND_CONTEXT);
|
||||
expect(late.ok).toBe(true);
|
||||
expect(await lane.drive({ operationId: "cancelled" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { status: "aborted" } },
|
||||
});
|
||||
expect(runtimeLane.state.inbox).toHaveLength(1);
|
||||
|
||||
faux.setResponses([fauxAssistantMessage("late answer")]);
|
||||
expect(await lane.prompt("", undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "run", status: "completed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("composes standalone compaction acceptance with drive", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
await lane.appendMessage({ role: "user", content: "history", timestamp: 1 }, BACKGROUND_CONTEXT);
|
||||
faux.setResponses([fauxAssistantMessage("summary")]);
|
||||
|
||||
expect(await lane.compact(undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { compaction: { kind: "compaction", status: "completed" } },
|
||||
});
|
||||
expect(faux.state.callCount).toBe(1);
|
||||
});
|
||||
|
||||
it.each([false, true])("composes %s summarized navigation acceptance with drive", async (summarize) => {
|
||||
const { lane, session, faux } = await createFixture();
|
||||
const rootId = await lane.appendMessage({ role: "user", content: "root", timestamp: 1 }, BACKGROUND_CONTEXT);
|
||||
await lane.appendMessage({ role: "user", content: "source", timestamp: 2 }, BACKGROUND_CONTEXT);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "target",
|
||||
parentId: rootId,
|
||||
type: "message",
|
||||
message: { role: "user", content: "target", timestamp: 3 },
|
||||
},
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
if (summarize) faux.setResponses([fauxAssistantMessage("branch summary")]);
|
||||
|
||||
expect(await lane.navigateTree("target", { summarize, label: "chosen" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { navigation: { kind: "navigation", status: "completed" } },
|
||||
});
|
||||
expect(faux.state.callCount).toBe(summarize ? 1 : 0);
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).not.toBeNull();
|
||||
});
|
||||
|
||||
it("resumes any current operation after acceptance or reopen", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
await acceptRun(lane, "run");
|
||||
faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
|
||||
expect(await lane.resume(BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { operationId: "run", kind: "run", status: "completed" },
|
||||
});
|
||||
});
|
||||
|
||||
it("polls one deferred permit through resume", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture({ deferred: true });
|
||||
faux.setResponses([fauxAssistantMessage("eventual answer")]);
|
||||
const suspended = await lane.prompt("defer", undefined, BACKGROUND_CONTEXT);
|
||||
if (!suspended.ok || suspended.value.status !== "suspended") throw new Error("Expected suspended run");
|
||||
|
||||
expect(await lane.resume(BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { operationId: suspended.value.operationId, kind: "run", status: "completed" },
|
||||
});
|
||||
expect(runtimeLane.state.operation).toBeNull();
|
||||
expect(faux.state.deferredFetchCount).toBe(1);
|
||||
expect(await lane.resume(BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "NothingToResume" },
|
||||
});
|
||||
});
|
||||
|
||||
it("aborts and reconciles the current operation", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
await acceptRun(lane, "run");
|
||||
|
||||
expect(await lane.abort(BACKGROUND_CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
value: { operationId: "run", steer: [], followUp: [] },
|
||||
});
|
||||
expect(runtimeLane.state.operation).toBeNull();
|
||||
expect(faux.state.callCount).toBe(0);
|
||||
expect(await lane.abort(BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "NoActiveOperation" },
|
||||
});
|
||||
});
|
||||
|
||||
it("waits for an operation that has no installed drive", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
await acceptRun(lane, "run");
|
||||
let idle = false;
|
||||
const waiting = lane.waitForIdle(BACKGROUND_CONTEXT).then(() => {
|
||||
idle = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(idle).toBe(false);
|
||||
|
||||
faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
await lane.drive({ operationId: "run" }, BACKGROUND_CONTEXT);
|
||||
await waiting;
|
||||
expect(idle).toBe(true);
|
||||
});
|
||||
|
||||
it("serializes concurrent runWhenIdle callbacks", async () => {
|
||||
const { lane } = await createFixture();
|
||||
const firstStarted = deferred();
|
||||
const releaseFirst = deferred();
|
||||
const order: string[] = [];
|
||||
const first = lane.runWhenIdle(async () => {
|
||||
order.push("first:start");
|
||||
firstStarted.resolve();
|
||||
await releaseFirst.promise;
|
||||
order.push("first:end");
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await firstStarted.promise;
|
||||
const second = lane.runWhenIdle(() => {
|
||||
order.push("second");
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await Promise.resolve();
|
||||
expect(order).toEqual(["first:start"]);
|
||||
|
||||
releaseFirst.resolve();
|
||||
await Promise.all([first, second]);
|
||||
expect(order).toEqual(["first:start", "first:end", "second"]);
|
||||
});
|
||||
|
||||
it("owns the idle window while runWhenIdle executes", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
const callbackStarted = deferred();
|
||||
const releaseCallback = deferred();
|
||||
const callback = lane.runWhenIdle(async () => {
|
||||
callbackStarted.resolve();
|
||||
await releaseCallback.promise;
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await callbackStarted.promise;
|
||||
let accepted = false;
|
||||
const acceptance = lane.accept({ kind: "prompt", prompt: "after" }, BACKGROUND_CONTEXT).then((result) => {
|
||||
accepted = true;
|
||||
return result;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(accepted).toBe(false);
|
||||
|
||||
releaseCallback.resolve();
|
||||
await callback;
|
||||
const admission = await acceptance;
|
||||
if (!admission.ok) throw admission.error;
|
||||
faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
await lane.drive({ operationId: admission.value.operationId }, BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("allows coherent lane reads from an idle callback", async () => {
|
||||
const { lane } = await createFixture();
|
||||
|
||||
await lane.runWhenIdle(async (context) => {
|
||||
const execution = await lane.inspectExecution(context);
|
||||
const watch = await lane.watch(context);
|
||||
expect(execution.current).toBeNull();
|
||||
expect(watch.snapshot.operation).toBeNull();
|
||||
watch.unsubscribe();
|
||||
}, BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("releases idle ownership when the callback fails", async () => {
|
||||
const { lane } = await createFixture();
|
||||
const failure = new Error("callback failed");
|
||||
|
||||
await expect(
|
||||
lane.runWhenIdle(() => {
|
||||
throw failure;
|
||||
}, BACKGROUND_CONTEXT),
|
||||
).rejects.toBe(failure);
|
||||
expect(await lane.accept({ kind: "prompt", prompt: "after" }, BACKGROUND_CONTEXT)).toMatchObject({ ok: true });
|
||||
});
|
||||
|
||||
it("close waits for an already-running idle callback", async () => {
|
||||
const { lane, harness } = await createFixture();
|
||||
const callbackStarted = deferred();
|
||||
const releaseCallback = deferred();
|
||||
const callback = lane.runWhenIdle(async () => {
|
||||
callbackStarted.resolve();
|
||||
await releaseCallback.promise;
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await callbackStarted.promise;
|
||||
let closed = false;
|
||||
const closing = harness.close(BACKGROUND_CONTEXT).then(() => {
|
||||
closed = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(closed).toBe(false);
|
||||
|
||||
releaseCallback.resolve();
|
||||
await Promise.all([callback, closing]);
|
||||
expect(closed).toBe(true);
|
||||
});
|
||||
|
||||
it("installs one pass and joins same-operation callers", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
faux.setResponses([
|
||||
async () => {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
return fauxAssistantMessage("answer");
|
||||
},
|
||||
]);
|
||||
const admission = await acceptRun(lane, "run");
|
||||
|
||||
const first = lane.drive({ operationId: admission.operationId }, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
const second = lane.drive({ operationId: admission.operationId }, BACKGROUND_CONTEXT);
|
||||
expect(runtimeLane.activeDrive?.operationId).toBe(admission.operationId);
|
||||
release.resolve();
|
||||
|
||||
const [firstResult, secondResult] = await Promise.all([first, second]);
|
||||
expect(firstResult).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { operationId: "run", status: "completed" } },
|
||||
});
|
||||
expect(secondResult).toEqual(firstResult);
|
||||
expect(faux.state.callCount).toBe(1);
|
||||
expect(runtimeLane.activeDrive).toBeUndefined();
|
||||
});
|
||||
|
||||
it("returns old result records without disturbing the current operation", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
faux.setResponses([fauxAssistantMessage("first")]);
|
||||
await acceptRun(lane, "first");
|
||||
expect(await lane.drive({ operationId: "first" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { operationId: "first" } },
|
||||
});
|
||||
await acceptRun(lane, "second");
|
||||
|
||||
expect(await lane.drive({ operationId: "first" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { operationId: "first" } },
|
||||
});
|
||||
expect(runtimeLane.state.operation?.meta.operationId).toBe("second");
|
||||
expect(runtimeLane.activeDrive).toBeUndefined();
|
||||
|
||||
faux.setResponses([fauxAssistantMessage("second")]);
|
||||
expect(await lane.drive({ operationId: "second" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { operationId: "second" } },
|
||||
});
|
||||
});
|
||||
|
||||
it("isolates stale operation ids", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
await acceptRun(lane, "current");
|
||||
|
||||
expect(await lane.drive({ operationId: "stale" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "OperationMismatch", expectedOperationId: "stale", currentOperationId: "current" },
|
||||
});
|
||||
expect(runtimeLane.activeDrive).toBeUndefined();
|
||||
expect(faux.state.callCount).toBe(0);
|
||||
});
|
||||
|
||||
it("does not install for a caller already cancelled", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
await acceptRun(lane, "run");
|
||||
const caller = withCancel(BACKGROUND_CONTEXT);
|
||||
const cancelled = new Error("caller cancelled");
|
||||
caller.cancel(cancelled);
|
||||
|
||||
await expect(lane.drive({ operationId: "run" }, caller.context)).rejects.toBe(cancelled);
|
||||
expect(runtimeLane.activeDrive).toBeUndefined();
|
||||
expect(faux.state.callCount).toBe(0);
|
||||
});
|
||||
|
||||
it("caller cancellation stops only that caller's observation", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
faux.setResponses([
|
||||
async () => {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
return fauxAssistantMessage("answer");
|
||||
},
|
||||
]);
|
||||
await acceptRun(lane, "run");
|
||||
const owner = lane.drive({ operationId: "run" }, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
const caller = withCancel(BACKGROUND_CONTEXT);
|
||||
const observer = lane.drive({ operationId: "run" }, caller.context);
|
||||
const cancelled = new Error("observer cancelled");
|
||||
caller.cancel(cancelled);
|
||||
|
||||
await expect(observer).rejects.toBe(cancelled);
|
||||
expect(runtimeLane.activeDrive?.operationId).toBe("run");
|
||||
release.resolve();
|
||||
expect(await owner).toMatchObject({ ok: true, value: { kind: "settled" } });
|
||||
expect(faux.state.callCount).toBe(1);
|
||||
});
|
||||
|
||||
it("close rejects observation without waiting for a non-cooperative effect", async () => {
|
||||
const { lane, runtimeLane, harness, faux } = await createFixture();
|
||||
const started = deferred();
|
||||
const release = deferred();
|
||||
faux.setResponses([
|
||||
async () => {
|
||||
started.resolve();
|
||||
await release.promise;
|
||||
return fauxAssistantMessage("late");
|
||||
},
|
||||
]);
|
||||
await acceptRun(lane, "run");
|
||||
const observation = lane.drive({ operationId: "run" }, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
|
||||
await harness.close(BACKGROUND_CONTEXT);
|
||||
await expect(observation).rejects.toBeInstanceOf(HarnessClosed);
|
||||
release.resolve();
|
||||
await expect.poll(() => runtimeLane.activeDrive).toBeUndefined();
|
||||
});
|
||||
|
||||
it("exposes durable abort and reconciles through public drive", async () => {
|
||||
const { lane, runtimeLane, faux } = await createFixture();
|
||||
await acceptRun(lane, "run");
|
||||
|
||||
expect(await lane.requestAbort("run", BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { operationId: "run", newlyRequested: true, steer: [], followUp: [] },
|
||||
});
|
||||
expect(runtimeLane.activeDrive).toBeUndefined();
|
||||
expect(await lane.drive({ operationId: "run" }, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "settled", outcome: { operationId: "run", status: "aborted" } },
|
||||
});
|
||||
expect(faux.state.callCount).toBe(0);
|
||||
expect(await lane.requestAbort("run", BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: false,
|
||||
error: { _tag: "OperationMismatch", expectedOperationId: "run", lastOperationId: "run" },
|
||||
});
|
||||
});
|
||||
|
||||
it("faults the harness when a detached pass fails", async () => {
|
||||
const { lane, runtimeLane, harness } = await createFixture();
|
||||
harness.hooks.on("before_drive", () => {
|
||||
throw new Error("drive failed");
|
||||
});
|
||||
await acceptRun(lane, "run");
|
||||
|
||||
await expect(lane.drive({ operationId: "run" }, BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(HarnessFault);
|
||||
expect(runtimeLane.activeDrive).toBeUndefined();
|
||||
await expect(lane.getTipId(BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(HarnessFault);
|
||||
});
|
||||
});
|
||||
747
third_party/pi/packages/agent/test/harness/runtime/drive-reconcile.test.ts
vendored
Normal file
747
third_party/pi/packages/agent/test/harness/runtime/drive-reconcile.test.ts
vendored
Normal file
@@ -0,0 +1,747 @@
|
||||
import {
|
||||
createModels,
|
||||
type DeferredHandle,
|
||||
fauxAssistantMessage,
|
||||
fauxProvider,
|
||||
fauxToolCall,
|
||||
type MutableModels,
|
||||
type Provider,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { HarnessEvent, WatchHandle } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { HookRegistry } from "../../../src/harness/hooks.ts";
|
||||
import { reconcileOperation } from "../../../src/harness/runtime/drive/reconcile.ts";
|
||||
import { runStructuralDecision } from "../../../src/harness/runtime/drive/structural.ts";
|
||||
import { driveOperation } from "../../../src/harness/runtime/drive.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { restoreLane } from "../../../src/harness/runtime/restore.ts";
|
||||
import { type Config, Drive } from "../../../src/harness/runtime/types.ts";
|
||||
import { insertEntry } from "../../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import { InstrumentedStorage } from "../../../src/harness/session/testing/instrumented-storage.ts";
|
||||
import type {
|
||||
Control,
|
||||
DeferredEffectPendingOperation,
|
||||
DeferredSuspendedOperation,
|
||||
LaneConfiguration,
|
||||
NewEntry,
|
||||
OperationMeta,
|
||||
OperationScope,
|
||||
OperationState,
|
||||
Session,
|
||||
SummaryContext,
|
||||
SummaryTask,
|
||||
Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import type { AgentMessage } from "../../../src/types.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const operationId = "01950000-0000-7000-8000-000000000001";
|
||||
|
||||
interface Fixture {
|
||||
lane: Lane<undefined>;
|
||||
drive: Drive;
|
||||
session: Session;
|
||||
storage: InstrumentedStorage;
|
||||
models: MutableModels;
|
||||
faux: ReturnType<typeof fauxProvider>;
|
||||
configuration: LaneConfiguration;
|
||||
events: HarnessEvent[];
|
||||
hooks: HookRegistry;
|
||||
}
|
||||
|
||||
interface InstalledOperation {
|
||||
state: OperationState;
|
||||
intent: OperationMeta["intent"];
|
||||
entries?: NewEntry[];
|
||||
writes?: Write[];
|
||||
terminalEvents: HarnessEvent["type"][];
|
||||
}
|
||||
|
||||
function unusedWatch<T>(): WatchHandle<T> {
|
||||
throw new Error("watch is not used by reconciliation tests");
|
||||
}
|
||||
|
||||
function cancelledControl(): Control {
|
||||
return { status: "cancel_requested", requestedAt: 10 };
|
||||
}
|
||||
|
||||
function scope(control: Control = cancelledControl()): OperationScope {
|
||||
return {
|
||||
control,
|
||||
settings: {
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
},
|
||||
latestAssistantEntryId: null,
|
||||
};
|
||||
}
|
||||
|
||||
function user(content: string): AgentMessage {
|
||||
return { role: "user", content, timestamp: 1 };
|
||||
}
|
||||
|
||||
function summaryContext(configuration: LaneConfiguration): SummaryContext {
|
||||
return {
|
||||
resultEntryId: "summary-entry",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
};
|
||||
}
|
||||
|
||||
async function createFixture(): Promise<Fixture> {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => 100 }));
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `reconcile-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
const faux = fauxProvider({ api: "faux", deferred: { pendingFetches: 0 } });
|
||||
const model = faux.getModel();
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: model.provider, modelId: model.id },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
};
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const hooks = new HookRegistry(() => {});
|
||||
const events: HarnessEvent[] = [];
|
||||
const config: Config<undefined> = {
|
||||
tools: [],
|
||||
resources: {},
|
||||
streamOptions: {},
|
||||
retryPolicy: { enabled: true, maxRetries: 1, baseDelayMs: 10 },
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
toolContext: undefined,
|
||||
systemPrompt: undefined,
|
||||
toProviderMessages: (messages) =>
|
||||
messages.filter(
|
||||
(message) => message.role === "user" || message.role === "assistant" || message.role === "toolResult",
|
||||
),
|
||||
entryProjectors: {},
|
||||
};
|
||||
const lane = new Lane<undefined>(
|
||||
"main",
|
||||
session,
|
||||
models,
|
||||
hooks,
|
||||
await restoreLane(session, "main", BACKGROUND_CONTEXT),
|
||||
(cause) => (cause instanceof Error ? cause : new Error(String(cause))),
|
||||
(batch) => {
|
||||
events.push(...structuredClone(batch));
|
||||
return Promise.resolve();
|
||||
},
|
||||
unusedWatch,
|
||||
() => config,
|
||||
);
|
||||
const drive = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
lane.activeDrive = drive;
|
||||
storage.clearCommitAttempts();
|
||||
return { lane, drive, session, storage, models, faux, configuration, events, hooks };
|
||||
}
|
||||
|
||||
async function installOperation(fixture: Fixture, installed: InstalledOperation): Promise<void> {
|
||||
const entries = installed.entries ?? [
|
||||
{ id: "tip", parentId: null, type: "message" as const, message: user("history") },
|
||||
];
|
||||
const tipId = entries.at(-1)?.id ?? null;
|
||||
const meta: OperationMeta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: tipId,
|
||||
startedAt: 1,
|
||||
intent: installed.intent,
|
||||
};
|
||||
await fixture.lane.command(
|
||||
(projection) => ({
|
||||
kind: "commit",
|
||||
writes: [
|
||||
...entries.map((entry) => insertEntry(entry)),
|
||||
storedValues.setValue(storedValues.branchTip("main"), tipId),
|
||||
...(installed.writes ?? []),
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), installed.state),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: projection.lastOperationId,
|
||||
inbox: projection.inbox,
|
||||
}),
|
||||
],
|
||||
next: { ...projection, tipId, operation: { meta, state: installed.state } },
|
||||
materialize: () => undefined,
|
||||
}),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
}
|
||||
|
||||
function deferredOperation(
|
||||
fixture: Fixture,
|
||||
effectPending: boolean,
|
||||
): { state: DeferredSuspendedOperation | DeferredEffectPendingOperation; entry: NewEntry; handle: DeferredHandle } {
|
||||
const model = fixture.faux.getModel();
|
||||
const handle = { provider: model.provider, modelId: model.id, api: model.api, id: "deferred-job" };
|
||||
const entry: NewEntry = {
|
||||
id: "deferred-source",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: fauxAssistantMessage([], { stopReason: "deferred", deferred: handle }),
|
||||
};
|
||||
const base = {
|
||||
...scope(),
|
||||
stepId: "step",
|
||||
sourceEntryId: entry.id,
|
||||
poll: effectPending ? 1 : 0,
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
};
|
||||
return {
|
||||
state: effectPending
|
||||
? {
|
||||
...base,
|
||||
at: "deferred.effect_pending",
|
||||
responseEntryId: "deferred-response",
|
||||
usageId: "deferred-usage",
|
||||
}
|
||||
: { ...base, at: "deferred.suspended" },
|
||||
entry,
|
||||
handle,
|
||||
};
|
||||
}
|
||||
|
||||
function cases(fixture: Fixture): InstalledOperation[] {
|
||||
const generation = {
|
||||
stepId: "step",
|
||||
triggerEntryId: "tip",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
overflowRecoveryUsed: false,
|
||||
};
|
||||
const runTask: SummaryTask = {
|
||||
taskId: "run-summary",
|
||||
reason: "threshold",
|
||||
boundary: {
|
||||
kind: "resume_checkpoint",
|
||||
resumeAfter: {
|
||||
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
|
||||
triggerEntryId: "tip",
|
||||
},
|
||||
},
|
||||
};
|
||||
const compactionTask: SummaryTask = { taskId: "compaction", reason: "manual", boundary: { kind: "finish" } };
|
||||
const navigationTask: SummaryTask = {
|
||||
taskId: "navigation",
|
||||
boundary: { kind: "commit_navigation", targetId: "target" },
|
||||
};
|
||||
const suspended = deferredOperation(fixture, false);
|
||||
const deferredEffect = deferredOperation(fixture, true);
|
||||
return [
|
||||
{
|
||||
state: { ...scope(), at: "starting" },
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "checkpoint",
|
||||
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
|
||||
triggerEntryId: "tip",
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: { ...scope(), at: "assistant.ready", generationContext: generation, nextAttempt: 1 },
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: generation,
|
||||
attempt: 1,
|
||||
responseEntryId: "assistant-response",
|
||||
usageId: "assistant-usage",
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1_000,
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
writes: [
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, "assistant-response"), {
|
||||
type: "text_delta",
|
||||
contentIndex: 0,
|
||||
delta: "partial",
|
||||
}),
|
||||
],
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "assistant.retry_wait",
|
||||
generationContext: generation,
|
||||
nextAttempt: 2,
|
||||
notBefore: Date.now() + 100_000,
|
||||
errorMessage: "retry",
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "tools",
|
||||
batch: {
|
||||
assistantEntryId: "assistant",
|
||||
configuration: fixture.configuration,
|
||||
turnId: "turn",
|
||||
calls: [{ status: "planned", sourceIndex: 0, resultEntryId: "tool-result" }],
|
||||
},
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
entries: [
|
||||
{
|
||||
id: "assistant",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: fauxAssistantMessage(fauxToolCall("tool", {}), { stopReason: "toolUse" }),
|
||||
},
|
||||
],
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: suspended.state,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
entries: [suspended.entry],
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: deferredEffect.state,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
entries: [deferredEffect.entry],
|
||||
terminalEvents: ["run_end"],
|
||||
},
|
||||
{
|
||||
state: { ...scope(), at: "summary.deciding", task: runTask },
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["compaction_end", "run_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "summary.ready",
|
||||
task: compactionTask,
|
||||
summaryContext: summaryContext(fixture.configuration),
|
||||
nextAttempt: 1,
|
||||
},
|
||||
intent: { kind: "compaction" },
|
||||
terminalEvents: ["compaction_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "summary.effect_pending",
|
||||
task: navigationTask,
|
||||
summaryContext: summaryContext(fixture.configuration),
|
||||
attempt: 1,
|
||||
request: { index: 0, usageId: "usage" },
|
||||
usageIds: [],
|
||||
},
|
||||
intent: { kind: "navigation", targetId: "target", summarize: true },
|
||||
terminalEvents: ["navigation_end"],
|
||||
},
|
||||
{
|
||||
state: {
|
||||
...scope(),
|
||||
at: "summary.retry_wait",
|
||||
task: runTask,
|
||||
summaryContext: summaryContext(fixture.configuration),
|
||||
nextAttempt: 2,
|
||||
notBefore: Date.now() + 100_000,
|
||||
errorMessage: "retry",
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["compaction_end", "run_end"],
|
||||
},
|
||||
{
|
||||
state: { ...scope(), at: "navigation.ready_to_commit", targetId: "target" },
|
||||
intent: { kind: "navigation", targetId: "target", summarize: false },
|
||||
terminalEvents: ["navigation_end"],
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.useRealTimers();
|
||||
vi.restoreAllMocks();
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime total drive", () => {
|
||||
it("drives an ordinary run through all direct procedures with one before_drive hook", async () => {
|
||||
const fixture = await createFixture();
|
||||
await installOperation(fixture, {
|
||||
state: { ...scope({ status: "running" }), at: "starting" },
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
});
|
||||
const beforeDrive = vi.fn();
|
||||
fixture.hooks.on("before_drive", beforeDrive);
|
||||
fixture.faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
|
||||
expect(await driveOperation(fixture.lane, fixture.drive)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, kind: "run", status: "completed" },
|
||||
});
|
||||
expect(beforeDrive).toHaveBeenCalledTimes(1);
|
||||
expect(fixture.faux.state.callCount).toBe(1);
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
});
|
||||
|
||||
it("leaves durable state unchanged when before_drive fails closed", async () => {
|
||||
const fixture = await createFixture();
|
||||
const starting = { ...scope({ status: "running" }), at: "starting" } as const;
|
||||
await installOperation(fixture, {
|
||||
state: starting,
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
});
|
||||
fixture.hooks.on("before_drive", () => {
|
||||
throw new Error("blocked drive");
|
||||
});
|
||||
|
||||
await expect(driveOperation(fixture.lane, fixture.drive)).rejects.toThrow("blocked drive");
|
||||
expect(fixture.lane.state.operation?.state).toEqual(starting);
|
||||
expect(fixture.storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime cancellation reconciliation", () => {
|
||||
it("reconciles every durable leaf without ordinary hook admission", async () => {
|
||||
for (let index = 0; index < 13; index++) {
|
||||
const fixture = await createFixture();
|
||||
const installed = cases(fixture)[index]!;
|
||||
await installOperation(fixture, installed);
|
||||
const beforeDrive = vi.fn();
|
||||
fixture.hooks.on("before_drive", beforeDrive);
|
||||
|
||||
const outcome = await driveOperation(fixture.lane, fixture.drive);
|
||||
|
||||
expect(outcome).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, status: "aborted" },
|
||||
});
|
||||
expect(beforeDrive).not.toHaveBeenCalled();
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
expect(await fixture.lane.getResult(operationId, BACKGROUND_CONTEXT)).toMatchObject({ status: "aborted" });
|
||||
expect(
|
||||
await fixture.session.getValue(storedValues.operationMeta(operationId), BACKGROUND_CONTEXT),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
await fixture.session.getValue(storedValues.operationState(operationId), BACKGROUND_CONTEXT),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
await fixture.session.scanValues(storedValues.operationToolArgsPrefix(operationId), BACKGROUND_CONTEXT),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await fixture.session.scanValues(storedValues.operationToolMemoPrefix(operationId), BACKGROUND_CONTEXT),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await fixture.session.scanValues(storedValues.operationPreparationPrefix(operationId), BACKGROUND_CONTEXT),
|
||||
).toEqual([]);
|
||||
if (installed.state.at === "deferred.suspended" || installed.state.at === "deferred.effect_pending") {
|
||||
expect(fixture.faux.state.cancelledDeferred).toHaveLength(1);
|
||||
}
|
||||
expect(
|
||||
fixture.events
|
||||
.filter((event) => installed.terminalEvents.includes(event.type))
|
||||
.slice(-installed.terminalEvents.length)
|
||||
.map((event) => event.type),
|
||||
).toEqual(installed.terminalEvents);
|
||||
}
|
||||
});
|
||||
|
||||
it("durably drains abortable input once and preserves lane-owned input through terminal cleanup", async () => {
|
||||
const fixture = await createFixture();
|
||||
await installOperation(fixture, {
|
||||
state: {
|
||||
...scope({ status: "running" }),
|
||||
at: "checkpoint",
|
||||
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
|
||||
triggerEntryId: "tip",
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
});
|
||||
const queued = [
|
||||
{ entryId: "steer", kind: "steer" as const, message: user("steer") },
|
||||
{ entryId: "write", kind: "write" as const, message: user("write") },
|
||||
{ entryId: "follow", kind: "followUp" as const, message: user("follow") },
|
||||
{ entryId: "next", kind: "nextRun" as const, message: user("next") },
|
||||
];
|
||||
await fixture.lane.command(
|
||||
(state) => ({
|
||||
kind: "commit",
|
||||
writes: [
|
||||
...queued.map((item) =>
|
||||
storedValues.setValue(storedValues.pendingEntry(item.entryId), {
|
||||
type: "message" as const,
|
||||
payload: item.message,
|
||||
}),
|
||||
),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: queued.map(({ entryId, kind }) => ({ entryId, kind })),
|
||||
}),
|
||||
],
|
||||
next: { ...state, inbox: queued.map(({ entryId, kind }) => ({ entryId, kind })) },
|
||||
materialize: () => undefined,
|
||||
}),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
const requested = await fixture.lane.requestOperationAbort(operationId, BACKGROUND_CONTEXT);
|
||||
expect(requested).toMatchObject({
|
||||
ok: true,
|
||||
value: {
|
||||
operationId,
|
||||
newlyRequested: true,
|
||||
steer: [{ content: "steer" }],
|
||||
followUp: [{ content: "follow" }],
|
||||
},
|
||||
});
|
||||
expect(fixture.drive.gate.signal.aborted).toBe(true);
|
||||
expect(fixture.drive.closeSignal.aborted).toBe(false);
|
||||
expect(fixture.lane.state.inbox).toEqual([
|
||||
{ entryId: "write", kind: "write" },
|
||||
{ entryId: "next", kind: "nextRun" },
|
||||
]);
|
||||
expect(await fixture.session.getValue(storedValues.pendingEntry("steer"), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(await fixture.session.getValue(storedValues.pendingEntry("follow"), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(fixture.events.filter((event) => event.type === "operation_abort")).toMatchObject([
|
||||
{ operationId, steer: [{ content: "steer" }], followUp: [{ content: "follow" }] },
|
||||
]);
|
||||
const commitCount = fixture.storage.getCommitAttempts().length;
|
||||
const eventCount = fixture.events.length;
|
||||
expect(await fixture.lane.requestOperationAbort(operationId, BACKGROUND_CONTEXT)).toEqual({
|
||||
ok: true,
|
||||
value: { operationId, newlyRequested: false, steer: [], followUp: [] },
|
||||
});
|
||||
expect(fixture.storage.getCommitAttempts()).toHaveLength(commitCount);
|
||||
expect(fixture.events).toHaveLength(eventCount);
|
||||
|
||||
expect(await driveOperation(fixture.lane, fixture.drive)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { status: "aborted" },
|
||||
});
|
||||
expect(fixture.lane.state.inbox).toEqual([
|
||||
{ entryId: "write", kind: "write" },
|
||||
{ entryId: "next", kind: "nextRun" },
|
||||
]);
|
||||
expect(await fixture.session.getValue(storedValues.pendingEntry("write"), BACKGROUND_CONTEXT)).toBeDefined();
|
||||
expect(await fixture.session.getValue(storedValues.pendingEntry("next"), BACKGROUND_CONTEXT)).toBeDefined();
|
||||
fixture.storage.clearCommitAttempts();
|
||||
const stale = await fixture.lane.requestOperationAbort(operationId, BACKGROUND_CONTEXT);
|
||||
expect(stale.ok).toBe(false);
|
||||
if (stale.ok) throw new Error("settled abort unexpectedly succeeded");
|
||||
expect(stale.error._tag).toBe("OperationMismatch");
|
||||
expect(fixture.storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it("marks cancellation without installing a Drive", async () => {
|
||||
const fixture = await createFixture();
|
||||
await installOperation(fixture, {
|
||||
state: { ...scope({ status: "running" }), at: "starting" },
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
});
|
||||
fixture.lane.activeDrive = undefined;
|
||||
|
||||
expect(await fixture.lane.requestOperationAbort(operationId, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { newlyRequested: true },
|
||||
});
|
||||
expect(fixture.lane.activeDrive).toBeUndefined();
|
||||
expect(fixture.lane.state.operation?.state.control.status).toBe("cancel_requested");
|
||||
});
|
||||
|
||||
it("cancels an admitted retry timer only after the abort marker commits", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(1_000);
|
||||
const fixture = await createFixture();
|
||||
const generation = {
|
||||
stepId: "step",
|
||||
triggerEntryId: "tip",
|
||||
configuration: fixture.configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 10 },
|
||||
overflowRecoveryUsed: false,
|
||||
};
|
||||
await installOperation(fixture, {
|
||||
state: {
|
||||
...scope({ status: "running" }),
|
||||
at: "assistant.retry_wait",
|
||||
generationContext: generation,
|
||||
nextAttempt: 2,
|
||||
notBefore: 2_000,
|
||||
errorMessage: "retry",
|
||||
},
|
||||
intent: { kind: "run", promptEntryIds: ["tip"] },
|
||||
terminalEvents: ["run_end"],
|
||||
});
|
||||
const drive = new Drive({ operationId, waitForRetry: true }, BACKGROUND_CONTEXT);
|
||||
fixture.lane.activeDrive = drive;
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
const running = driveOperation(fixture.lane, drive);
|
||||
await vi.waitFor(() => expect(setTimeoutSpy).toHaveBeenCalled());
|
||||
|
||||
expect(await fixture.lane.requestOperationAbort(operationId, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { newlyRequested: true },
|
||||
});
|
||||
expect(await running).toMatchObject({ kind: "settled", outcome: { status: "aborted" } });
|
||||
expect(fixture.events.some((event) => event.type === "retry_start")).toBe(false);
|
||||
});
|
||||
|
||||
it("drops a stale structural hook result when cancellation commits first", async () => {
|
||||
const fixture = await createFixture();
|
||||
const task: SummaryTask = { taskId: "task", reason: "manual", boundary: { kind: "finish" } };
|
||||
const deciding = { ...scope({ status: "running" }), at: "summary.deciding", task } as const;
|
||||
await installOperation(fixture, {
|
||||
state: deciding,
|
||||
intent: { kind: "compaction" },
|
||||
writes: [
|
||||
storedValues.setValue(storedValues.operationPreparation(operationId, task.taskId), {
|
||||
kind: "compaction",
|
||||
messagesToSummarize: [user("history")],
|
||||
turnPrefixMessages: [],
|
||||
retainedTail: [],
|
||||
isSplitTurn: false,
|
||||
tokensBefore: 100,
|
||||
fileOps: { read: [], written: [], edited: [] },
|
||||
settings: DEFAULT_COMPACTION_SETTINGS,
|
||||
}),
|
||||
],
|
||||
terminalEvents: ["compaction_end"],
|
||||
});
|
||||
let releaseHook!: () => void;
|
||||
const hookStarted = new Promise<void>((resolve) => {
|
||||
fixture.hooks.on("before_compaction", async () => {
|
||||
resolve();
|
||||
await new Promise<void>((release) => {
|
||||
releaseHook = release;
|
||||
});
|
||||
return {
|
||||
compaction: { summary: "stale", tokensBefore: 100, retainedTail: [] },
|
||||
};
|
||||
});
|
||||
});
|
||||
const running = runStructuralDecision(fixture.lane, fixture.drive, deciding);
|
||||
await hookStarted;
|
||||
await fixture.lane.requestOperationAbort(operationId, BACKGROUND_CONTEXT);
|
||||
releaseHook();
|
||||
|
||||
expect(await running).toEqual({ kind: "continue" });
|
||||
expect(await fixture.session.findEntries({ type: "compaction" }, BACKGROUND_CONTEXT)).toEqual([]);
|
||||
expect(await driveOperation(fixture.lane, fixture.drive)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { status: "aborted" },
|
||||
});
|
||||
expect(fixture.events.at(-1)).toMatchObject({ type: "compaction_end", status: "aborted" });
|
||||
});
|
||||
|
||||
it("keeps the deferred cleanup signal separate from operation abort", () => {
|
||||
const drive = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
drive.beginAbort(Promise.resolve());
|
||||
drive.signalAbort();
|
||||
expect(drive.gate.signal.aborted).toBe(true);
|
||||
expect(drive.closeSignal.aborted).toBe(false);
|
||||
const closed = new Error("closed");
|
||||
drive.closeGate(closed);
|
||||
expect(drive.closeSignal.aborted).toBe(true);
|
||||
expect(drive.closeSignal.reason).toBe(closed);
|
||||
});
|
||||
|
||||
it("can crash between cancelled response settlement and terminal cleanup", async () => {
|
||||
const fixture = await createFixture();
|
||||
const installed = cases(fixture)[3]!;
|
||||
await installOperation(fixture, installed);
|
||||
|
||||
expect(await reconcileOperation(fixture.lane, fixture.drive)).toEqual({ kind: "continue" });
|
||||
expect(fixture.lane.state.operation?.state.at).toBe("checkpoint");
|
||||
expect(await fixture.session.getEntry("assistant-response", BACKGROUND_CONTEXT)).toMatchObject({
|
||||
type: "message",
|
||||
message: { stopReason: "aborted" },
|
||||
});
|
||||
expect(await fixture.lane.getResult(operationId, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(
|
||||
await fixture.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, "assistant-response"),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
|
||||
const resumed = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
fixture.lane.activeDrive = resumed;
|
||||
expect(await driveOperation(fixture.lane, resumed)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { status: "aborted" },
|
||||
});
|
||||
});
|
||||
|
||||
it("ignores deferred-provider cancellation failure", async () => {
|
||||
const fixture = await createFixture();
|
||||
const deferred = deferredOperation(fixture, false);
|
||||
const provider: Provider = {
|
||||
...fixture.faux.provider,
|
||||
cancelDeferred: async () => {
|
||||
throw new Error("remote cancellation failed");
|
||||
},
|
||||
};
|
||||
fixture.models.setProvider(provider);
|
||||
await installOperation(fixture, {
|
||||
state: deferred.state,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
entries: [deferred.entry],
|
||||
terminalEvents: ["run_end"],
|
||||
});
|
||||
|
||||
expect(await driveOperation(fixture.lane, fixture.drive)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { status: "aborted" },
|
||||
});
|
||||
});
|
||||
});
|
||||
613
third_party/pi/packages/agent/test/harness/runtime/drive-retry-deferred.test.ts
vendored
Normal file
613
third_party/pi/packages/agent/test/harness/runtime/drive-retry-deferred.test.ts
vendored
Normal file
@@ -0,0 +1,613 @@
|
||||
import {
|
||||
type AssistantMessage,
|
||||
type DeferredFetchOptions,
|
||||
fauxAssistantMessage,
|
||||
fauxProvider,
|
||||
fauxToolCall,
|
||||
type MutableModels,
|
||||
type Provider,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createModels } from "../../../../ai/src/models.ts";
|
||||
import type { HarnessEvent, WatchHandle } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { HookRegistry } from "../../../src/harness/hooks.ts";
|
||||
import { runCheckpoint, startRun } from "../../../src/harness/runtime/drive/checkpoint.ts";
|
||||
import { recoverDeferredPoll, runDeferred, runDeferredSuspended } from "../../../src/harness/runtime/drive/deferred.ts";
|
||||
import { runGeneration } from "../../../src/harness/runtime/drive/generation.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { restoreLane } from "../../../src/harness/runtime/restore.ts";
|
||||
import { type Config, Drive } from "../../../src/harness/runtime/types.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import { GatingStorage } from "../../../src/harness/session/testing/gating-storage.ts";
|
||||
import { InstrumentedStorage } from "../../../src/harness/session/testing/instrumented-storage.ts";
|
||||
import {
|
||||
type AssistantReadyOperation,
|
||||
type AssistantRetryWaitOperation,
|
||||
type DeferredEffectPendingOperation,
|
||||
type DeferredSuspendedOperation,
|
||||
type LaneConfiguration,
|
||||
type OperationState,
|
||||
operationScopeOf,
|
||||
type Session,
|
||||
type Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import { deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const operationId = "01950000-0000-7000-8000-000000000001";
|
||||
|
||||
interface Fixture {
|
||||
backend: MemoryStorage;
|
||||
gating: GatingStorage | undefined;
|
||||
storage: InstrumentedStorage;
|
||||
session: Session;
|
||||
lane: Lane<undefined>;
|
||||
drive: Drive;
|
||||
models: MutableModels;
|
||||
faux: ReturnType<typeof fauxProvider>;
|
||||
hooks: HookRegistry;
|
||||
events: HarnessEvent[];
|
||||
config: Config<undefined>;
|
||||
}
|
||||
|
||||
function unusedWatch<T>(): WatchHandle<T> {
|
||||
throw new Error("watch is not used by retry/deferred tests");
|
||||
}
|
||||
|
||||
async function createFixture(
|
||||
options: { gated?: boolean; pendingFetches?: number; deferredSubmission?: boolean } = {},
|
||||
): Promise<Fixture> {
|
||||
const backend = new MemoryStorage({ now: () => 100 });
|
||||
const gating = options.gated ? new GatingStorage(backend) : undefined;
|
||||
const storage = new InstrumentedStorage(gating ?? backend);
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `retry-deferred-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
const faux = fauxProvider({
|
||||
tokenSize: { min: 1, max: 1 },
|
||||
deferred: { pendingFetches: options.pendingFetches ?? 0 },
|
||||
});
|
||||
const model = faux.getModel();
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: model.provider, modelId: model.id },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
};
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
const hooks = new HookRegistry(() => {});
|
||||
const events: HarnessEvent[] = [];
|
||||
const config: Config<undefined> = {
|
||||
tools: [],
|
||||
resources: {},
|
||||
streamOptions: { deferred: options.deferredSubmission ?? true },
|
||||
retryPolicy: { enabled: true, maxRetries: 3, baseDelayMs: 10 },
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
toolContext: undefined,
|
||||
systemPrompt: undefined,
|
||||
toProviderMessages: (messages) =>
|
||||
messages.filter(
|
||||
(message) => message.role === "user" || message.role === "assistant" || message.role === "toolResult",
|
||||
),
|
||||
entryProjectors: {},
|
||||
};
|
||||
const lane = new Lane<undefined>(
|
||||
"main",
|
||||
session,
|
||||
models,
|
||||
hooks,
|
||||
await restoreLane(session, "main", BACKGROUND_CONTEXT),
|
||||
(cause) => (cause instanceof Error ? cause : new Error(String(cause))),
|
||||
(batch) => {
|
||||
events.push(...structuredClone(batch));
|
||||
return Promise.resolve();
|
||||
},
|
||||
unusedWatch,
|
||||
() => config,
|
||||
);
|
||||
const admission = await lane.accept({ kind: "prompt", operationId, prompt: "question" }, BACKGROUND_CONTEXT);
|
||||
if (!admission.ok) throw admission.error;
|
||||
const drive = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
lane.activeDrive = drive;
|
||||
storage.clearCommitAttempts();
|
||||
return { backend, gating, storage, session, lane, drive, models, faux, hooks, events, config };
|
||||
}
|
||||
|
||||
function currentRun(fixture: Fixture): OperationState {
|
||||
const operation = fixture.lane.state.operation;
|
||||
if (operation === null) throw new Error("fixture has no operation");
|
||||
return operation.state;
|
||||
}
|
||||
|
||||
async function advanceToReady(fixture: Fixture): Promise<AssistantReadyOperation> {
|
||||
const starting = currentRun(fixture);
|
||||
if (starting.at !== "starting") throw new Error("run did not start at its initial boundary");
|
||||
await startRun(fixture.lane, fixture.drive, starting);
|
||||
let run = currentRun(fixture);
|
||||
if (run.at !== "checkpoint") throw new Error("run did not reach checkpoint");
|
||||
await runCheckpoint(fixture.lane, fixture.drive, run);
|
||||
run = currentRun(fixture);
|
||||
if (run.at !== "assistant.ready") {
|
||||
throw new Error("run did not reach ready generation");
|
||||
}
|
||||
return run;
|
||||
}
|
||||
|
||||
async function replaceRunState(fixture: Fixture, nextState: OperationState, extraWrites: Write[] = []): Promise<void> {
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("fixture has no operation");
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [...extraWrites, storedValues.setValue(storedValues.operationState(operationId), nextState)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: nextState } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
async function submitDeferred(
|
||||
fixture: Fixture,
|
||||
response: AssistantMessage = fauxAssistantMessage("done", { timestamp: 20 }),
|
||||
): Promise<DeferredSuspendedOperation> {
|
||||
fixture.faux.setResponses([response]);
|
||||
const ready = await advanceToReady(fixture);
|
||||
await runGeneration(fixture.lane, fixture.drive, ready);
|
||||
const run = currentRun(fixture);
|
||||
if (run.at !== "deferred.suspended") {
|
||||
throw new Error("initial request did not suspend");
|
||||
}
|
||||
return run;
|
||||
}
|
||||
|
||||
async function installUnknownPoll(
|
||||
fixture: Fixture,
|
||||
suspended: DeferredSuspendedOperation,
|
||||
): Promise<DeferredEffectPendingOperation> {
|
||||
const effectPending: DeferredEffectPendingOperation = {
|
||||
...operationScopeOf(currentRun(fixture)),
|
||||
at: "deferred.effect_pending",
|
||||
stepId: suspended.stepId,
|
||||
sourceEntryId: suspended.sourceEntryId,
|
||||
poll: suspended.poll + 1,
|
||||
responseEntryId: fixture.session.idGenerator.next(),
|
||||
usageId: fixture.session.idGenerator.next(),
|
||||
configuration: suspended.configuration,
|
||||
streamOptions: suspended.streamOptions,
|
||||
};
|
||||
await replaceRunState(fixture, effectPending, [
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, effectPending.responseEntryId), {
|
||||
type: "text_delta",
|
||||
contentIndex: 0,
|
||||
delta: "old",
|
||||
}),
|
||||
]);
|
||||
return effectPending;
|
||||
}
|
||||
|
||||
function currentDeferred(fixture: Fixture): DeferredSuspendedOperation | DeferredEffectPendingOperation {
|
||||
const run = currentRun(fixture);
|
||||
if (run.at !== "deferred.suspended" && run.at !== "deferred.effect_pending") {
|
||||
throw new Error("fixture has no deferred phase");
|
||||
}
|
||||
return run;
|
||||
}
|
||||
|
||||
function installDrive(fixture: Fixture, options: { waitForRetry?: boolean; pollDeferred?: boolean }): Drive {
|
||||
const drive = new Drive({ operationId, ...options }, BACKGROUND_CONTEXT);
|
||||
fixture.lane.activeDrive = drive;
|
||||
return drive;
|
||||
}
|
||||
|
||||
async function expectProjectionRestores(fixture: Fixture): Promise<void> {
|
||||
expect(fixture.lane.state).toEqual(await restoreLane(fixture.session, "main", BACKGROUND_CONTEXT));
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.useRealTimers();
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime assistant retry wait", () => {
|
||||
it("classifies a live retryable provider error into durable retry wait", async () => {
|
||||
const fixture = await createFixture({ deferredSubmission: false });
|
||||
const ready = await advanceToReady(fixture);
|
||||
fixture.faux.setResponses([
|
||||
fauxAssistantMessage([], { stopReason: "error", errorMessage: "503 service unavailable", timestamp: 10 }),
|
||||
]);
|
||||
|
||||
expect(await runGeneration(fixture.lane, fixture.drive, ready)).toEqual({
|
||||
kind: "continue",
|
||||
});
|
||||
expect(currentRun(fixture)).toMatchObject({
|
||||
at: "assistant.retry_wait",
|
||||
nextAttempt: 2,
|
||||
errorMessage: "503 service unavailable",
|
||||
});
|
||||
expect(fixture.events.at(-1)).toMatchObject({ type: "retry_scheduled", attempt: 2 });
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("returns a durable waiting outcome without a timer or write when local waiting is disabled", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(1_000);
|
||||
const fixture = await createFixture({ deferredSubmission: false });
|
||||
const ready = await advanceToReady(fixture);
|
||||
const retryWait: AssistantRetryWaitOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.retry_wait",
|
||||
generationContext: ready.generationContext,
|
||||
nextAttempt: 2,
|
||||
notBefore: 1_100,
|
||||
errorMessage: "retry",
|
||||
};
|
||||
await replaceRunState(fixture, retryWait);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
const setTimeoutSpy = vi.spyOn(globalThis, "setTimeout");
|
||||
|
||||
expect(await runGeneration(fixture.lane, fixture.drive, retryWait)).toEqual({
|
||||
kind: "waiting",
|
||||
outcome: { kind: "waiting", operationId, reason: "retry", notBefore: 1_100 },
|
||||
});
|
||||
expect(setTimeoutSpy).not.toHaveBeenCalled();
|
||||
expect(fixture.storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it("commits ready at the deadline and emits retry lifecycle around the next attempt", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(2_000);
|
||||
const fixture = await createFixture({ deferredSubmission: false });
|
||||
const ready = await advanceToReady(fixture);
|
||||
const retryWait: AssistantRetryWaitOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.retry_wait",
|
||||
generationContext: ready.generationContext,
|
||||
nextAttempt: 2,
|
||||
notBefore: 2_000,
|
||||
errorMessage: "retry",
|
||||
};
|
||||
await replaceRunState(fixture, retryWait);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
expect(await runGeneration(fixture.lane, fixture.drive, retryWait)).toEqual({
|
||||
kind: "continue",
|
||||
});
|
||||
const next = currentRun(fixture);
|
||||
if (next.at !== "assistant.ready") {
|
||||
throw new Error("retry did not become ready");
|
||||
}
|
||||
expect(fixture.events.at(-1)).toMatchObject({ type: "retry_start", attempt: 2 });
|
||||
fixture.faux.setResponses([fauxAssistantMessage("retried", { timestamp: 30 })]);
|
||||
await runGeneration(fixture.lane, fixture.drive, next);
|
||||
expect(fixture.events).toContainEqual(expect.objectContaining({ type: "retry_end", attempt: 2, success: true }));
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("admits an abort-aware timer only for local waiting", async () => {
|
||||
vi.useFakeTimers();
|
||||
vi.setSystemTime(3_000);
|
||||
const fixture = await createFixture({ deferredSubmission: false });
|
||||
const ready = await advanceToReady(fixture);
|
||||
const retryWait: AssistantRetryWaitOperation = {
|
||||
...operationScopeOf(ready),
|
||||
at: "assistant.retry_wait",
|
||||
generationContext: ready.generationContext,
|
||||
nextAttempt: 2,
|
||||
notBefore: 3_100,
|
||||
errorMessage: "retry",
|
||||
};
|
||||
await replaceRunState(fixture, retryWait);
|
||||
const drive = installDrive(fixture, { waitForRetry: true });
|
||||
fixture.storage.clearCommitAttempts();
|
||||
const waiting = runGeneration(fixture.lane, drive, retryWait);
|
||||
|
||||
await vi.advanceTimersByTimeAsync(99);
|
||||
expect(fixture.storage.getCommitAttempts()).toEqual([]);
|
||||
await vi.advanceTimersByTimeAsync(1);
|
||||
expect(await waiting).toEqual({ kind: "continue" });
|
||||
expect(currentRun(fixture)).toMatchObject({
|
||||
at: "assistant.ready",
|
||||
nextAttempt: 2,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime deferred polling", () => {
|
||||
it("waits without a permit, then performs at most one pending poll from captured options", async () => {
|
||||
const fixture = await createFixture({ pendingFetches: 1 });
|
||||
const suspended = await submitDeferred(fixture);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
expect(await runDeferredSuspended(fixture.lane, fixture.drive, suspended)).toEqual({
|
||||
kind: "waiting",
|
||||
outcome: {
|
||||
kind: "waiting",
|
||||
operationId,
|
||||
reason: "deferred",
|
||||
deferred: expect.objectContaining({ id: expect.any(String) }),
|
||||
},
|
||||
});
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(0);
|
||||
expect(fixture.storage.getCommitAttempts()).toEqual([]);
|
||||
|
||||
let hookOptions: unknown;
|
||||
fixture.hooks.on("before_request", (event) => {
|
||||
if (event.step === "deferred") hookOptions = event.streamOptions;
|
||||
return undefined;
|
||||
});
|
||||
let fetchOptions: DeferredFetchOptions | undefined;
|
||||
const provider = fixture.faux.provider;
|
||||
const fetchDeferred = provider.fetchDeferred;
|
||||
if (fetchDeferred === undefined) throw new Error("faux provider has no deferred fetch");
|
||||
const wrapped: Provider = {
|
||||
...provider,
|
||||
fetchDeferred: (model, handle, options) => {
|
||||
fetchOptions = options;
|
||||
return fetchDeferred(model, handle, options);
|
||||
},
|
||||
};
|
||||
fixture.models.setProvider(wrapped);
|
||||
const driveOptions = { operationId, pollDeferred: true } as const;
|
||||
const drive = new Drive(driveOptions, BACKGROUND_CONTEXT);
|
||||
fixture.lane.activeDrive = drive;
|
||||
|
||||
expect(await runDeferred(fixture.lane, drive, currentDeferred(fixture))).toEqual({
|
||||
kind: "continue",
|
||||
});
|
||||
expect(drive.deferredPermits).toBe(0);
|
||||
expect(driveOptions).toEqual({ operationId, pollDeferred: true });
|
||||
expect(hookOptions).toMatchObject({ deferred: false });
|
||||
expect(fetchOptions).toMatchObject({ wait: 0 });
|
||||
expect(fetchOptions?.signal).toBe(drive.gate.signal);
|
||||
expect(currentDeferred(fixture)).toMatchObject({ at: "deferred.suspended", poll: 1 });
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(1);
|
||||
expect(fixture.events.slice(-2).map((event) => event.type)).toEqual(["turn_end", "run_suspend"]);
|
||||
|
||||
expect(await runDeferred(fixture.lane, drive, currentDeferred(fixture))).toMatchObject({
|
||||
kind: "waiting",
|
||||
outcome: { reason: "deferred" },
|
||||
});
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(1);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("declines poll intent when cancellation wins preparation", async () => {
|
||||
const fixture = await createFixture();
|
||||
const suspended = await submitDeferred(fixture);
|
||||
const drive = installDrive(fixture, { pollDeferred: true });
|
||||
const hookStarted = deferred();
|
||||
const releaseHook = deferred();
|
||||
fixture.hooks.on("before_request", async (event) => {
|
||||
if (event.step !== "deferred") return undefined;
|
||||
hookStarted.resolve();
|
||||
await releaseHook.promise;
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const polling = runDeferredSuspended(fixture.lane, drive, suspended);
|
||||
await hookStarted.promise;
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const nextState: OperationState = {
|
||||
...operation.state,
|
||||
control: { status: "cancel_requested", requestedAt: 10 },
|
||||
};
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(operationId), nextState)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: nextState } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
releaseHook.resolve();
|
||||
|
||||
expect(await polling).toEqual({ kind: "continue" });
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(0);
|
||||
expect(currentRun(fixture)).toMatchObject({
|
||||
at: "deferred.suspended",
|
||||
control: { status: "cancel_requested" },
|
||||
});
|
||||
});
|
||||
|
||||
it("plans ready deferred tool calls with the poll turn identity and follower result ids", async () => {
|
||||
const fixture = await createFixture();
|
||||
const suspended = await submitDeferred(
|
||||
fixture,
|
||||
fauxAssistantMessage(fauxToolCall("lookup", { query: "value" }), {
|
||||
stopReason: "toolUse",
|
||||
timestamp: 20,
|
||||
}),
|
||||
);
|
||||
const drive = installDrive(fixture, { pollDeferred: true });
|
||||
|
||||
expect(await runDeferredSuspended(fixture.lane, drive, suspended)).toEqual({
|
||||
kind: "continue",
|
||||
});
|
||||
const run = currentRun(fixture);
|
||||
if (run.at !== "tools") throw new Error("deferred tool response did not create a batch");
|
||||
expect(run.batch).toMatchObject({
|
||||
turnId: `${suspended.stepId}:poll:1`,
|
||||
calls: [{ status: "planned", sourceIndex: 0 }],
|
||||
});
|
||||
expect(run.batch.calls[0]?.resultEntryId.slice(0, 13)).toBe(run.latestAssistantEntryId?.slice(0, 13));
|
||||
expect(fixture.events.at(-1)).toMatchObject({ type: "usage" });
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("consumes its permit only after the fresh intent commit lands", async () => {
|
||||
const fixture = await createFixture({ gated: true });
|
||||
const suspended = await submitDeferred(
|
||||
fixture,
|
||||
fauxAssistantMessage([], { stopReason: "error", errorMessage: "failed", timestamp: 20 }),
|
||||
);
|
||||
const gating = fixture.gating;
|
||||
if (gating === undefined) throw new Error("fixture is not gated");
|
||||
const drive = installDrive(fixture, { pollDeferred: true });
|
||||
gating.arm();
|
||||
const polling = runDeferredSuspended(fixture.lane, drive, suspended);
|
||||
|
||||
await gating.waitPending();
|
||||
expect(drive.deferredPermits).toBe(1);
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(0);
|
||||
await gating.next();
|
||||
expect(drive.deferredPermits).toBe(0);
|
||||
await expect.poll(() => fixture.faux.state.deferredFetchCount).toBe(1);
|
||||
await gating.next(2); // start frame, then response settlement
|
||||
expect(await polling).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, kind: "run", status: "failed" },
|
||||
});
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
});
|
||||
|
||||
it("leaves suspended state unchanged when the fresh intent is discarded", async () => {
|
||||
const fixture = await createFixture({ gated: true });
|
||||
const suspended = await submitDeferred(fixture);
|
||||
const gating = fixture.gating;
|
||||
if (gating === undefined) throw new Error("fixture is not gated");
|
||||
const drive = installDrive(fixture, { pollDeferred: true });
|
||||
gating.arm();
|
||||
const polling = runDeferredSuspended(fixture.lane, drive, suspended);
|
||||
void polling.catch(() => {});
|
||||
|
||||
await gating.waitPending();
|
||||
gating.discard();
|
||||
await expect(polling).rejects.toThrow("commit discarded");
|
||||
expect(drive.deferredPermits).toBe(1);
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(0);
|
||||
const durable = await fixture.backend.getValue(storedValues.operationState(operationId), BACKGROUND_CONTEXT);
|
||||
expect(durable?.value).toMatchObject({ at: "deferred.suspended", poll: 0 });
|
||||
});
|
||||
|
||||
it("replaces an unknown poll under fresh ids at the same poll number and deletes old frames", async () => {
|
||||
const fixture = await createFixture();
|
||||
const suspended = await submitDeferred(fixture);
|
||||
const unknown = await installUnknownPoll(fixture, suspended);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
|
||||
const noPermit = installDrive(fixture, {});
|
||||
expect(await recoverDeferredPoll(fixture.lane, noPermit, unknown)).toMatchObject({
|
||||
kind: "waiting",
|
||||
outcome: { reason: "deferred" },
|
||||
});
|
||||
expect(
|
||||
await fixture.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, unknown.responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toHaveLength(1);
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(0);
|
||||
|
||||
const replacement = installDrive(fixture, { pollDeferred: true });
|
||||
expect(await recoverDeferredPoll(fixture.lane, replacement, unknown)).toEqual({
|
||||
kind: "continue",
|
||||
});
|
||||
const intent = fixture.storage
|
||||
.getCommitAttempts()
|
||||
.find((writes) =>
|
||||
writes.some(
|
||||
(write) =>
|
||||
write.kind === "list" &&
|
||||
write.op === "delete" &&
|
||||
write.namespace === "pi.pending.assistant_frame" &&
|
||||
write.key.endsWith(unknown.responseEntryId),
|
||||
),
|
||||
);
|
||||
expect(intent?.map((write) => `${write.kind}:${"op" in write ? write.op : "insert"}`)).toEqual([
|
||||
"list:delete",
|
||||
"value:set",
|
||||
]);
|
||||
const intentState = intent?.find(
|
||||
(write) => write.kind === "value" && write.op === "set" && write.namespace === "pi.op.state",
|
||||
);
|
||||
expect(intentState).toMatchObject({
|
||||
value: {
|
||||
at: "deferred.effect_pending",
|
||||
poll: unknown.poll,
|
||||
responseEntryId: expect.not.stringMatching(unknown.responseEntryId),
|
||||
usageId: expect.not.stringMatching(unknown.usageId),
|
||||
},
|
||||
});
|
||||
expect(await fixture.session.getEntry(unknown.responseEntryId, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect((await fixture.storage.scanUsage({}, BACKGROUND_CONTEXT)).some((row) => row.id === unknown.usageId)).toBe(
|
||||
false,
|
||||
);
|
||||
expect(
|
||||
await fixture.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, unknown.responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(currentRun(fixture).at).toBe("checkpoint");
|
||||
expect(fixture.events.some((event) => "recovery" in event && event.recovery === true)).toBe(true);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("abandons unknown ids and frames into configuration failure when the captured model is unavailable", async () => {
|
||||
const fixture = await createFixture();
|
||||
const suspended = await submitDeferred(fixture);
|
||||
const unknown = await installUnknownPoll(fixture, suspended);
|
||||
fixture.models.deleteProvider(fixture.faux.provider.id);
|
||||
fixture.storage.clearCommitAttempts();
|
||||
const drive = installDrive(fixture, { pollDeferred: true });
|
||||
|
||||
expect(await recoverDeferredPoll(fixture.lane, drive, unknown)).toMatchObject({
|
||||
kind: "settled",
|
||||
outcome: { operationId, kind: "run", status: "failed", error: { code: "model_unavailable" } },
|
||||
});
|
||||
expect(drive.deferredPermits).toBe(1);
|
||||
expect(fixture.faux.state.deferredFetchCount).toBe(0);
|
||||
expect(fixture.lane.state.operation).toBeNull();
|
||||
expect(
|
||||
fixture.storage
|
||||
.getCommitAttempts()
|
||||
.at(-1)
|
||||
?.map((write) =>
|
||||
write.kind === "value" || write.kind === "list"
|
||||
? `${write.kind}:${write.op}:${write.namespace}`
|
||||
: write.kind,
|
||||
),
|
||||
).toEqual([
|
||||
"value:delete:pi.op.meta",
|
||||
"value:delete:pi.op.state",
|
||||
"list:delete:pi.pending.assistant_frame",
|
||||
"value:set:pi.result",
|
||||
"value:set:pi.lane.state",
|
||||
]);
|
||||
expect(
|
||||
fixture.storage
|
||||
.getCommitAttempts()
|
||||
.flat()
|
||||
.some((write) => write.kind === "entry" || write.kind === "usage"),
|
||||
).toBe(false);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
});
|
||||
1451
third_party/pi/packages/agent/test/harness/runtime/drive-structural.test.ts
vendored
Normal file
1451
third_party/pi/packages/agent/test/harness/runtime/drive-structural.test.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
289
third_party/pi/packages/agent/test/harness/runtime/drive-terminal.test.ts
vendored
Normal file
289
third_party/pi/packages/agent/test/harness/runtime/drive-terminal.test.ts
vendored
Normal file
@@ -0,0 +1,289 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { operationCleanupWrites, operationResultRecord } from "../../../src/harness/runtime/drive/terminal.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type {
|
||||
AssistantEffectPendingOperation,
|
||||
LaneConfiguration,
|
||||
NavigationReadyToCommitOperation,
|
||||
OperationMeta,
|
||||
OperationScope,
|
||||
OperationState,
|
||||
Session,
|
||||
SummaryDecidingOperation,
|
||||
ToolsOperation,
|
||||
Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: "test", modelId: "model" },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
};
|
||||
|
||||
function runScope(): OperationScope {
|
||||
return {
|
||||
control: { status: "running" },
|
||||
settings: {
|
||||
compaction: { enabled: true, reserveTokens: 1_000, keepRecentTokens: 2_000 },
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
},
|
||||
latestAssistantEntryId: null,
|
||||
};
|
||||
}
|
||||
|
||||
function meta(operationId: string, state: OperationState): OperationMeta {
|
||||
const intent: OperationMeta["intent"] =
|
||||
state.at === "summary.deciding"
|
||||
? { kind: "compaction" }
|
||||
: state.at === "navigation.ready_to_commit"
|
||||
? { kind: "navigation", targetId: state.targetId, summarize: false }
|
||||
: { kind: "run", promptEntryIds: [] };
|
||||
return { operationId, lane: "main", sourceTipId: null, startedAt: 1, intent };
|
||||
}
|
||||
|
||||
async function createSession(): Promise<{ session: Session; storage: MemoryStorage }> {
|
||||
const storage = new MemoryStorage();
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `terminal-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
return { session, storage };
|
||||
}
|
||||
|
||||
async function commit(session: Session, writes: Write[]): Promise<void> {
|
||||
await session.mutate(async (mutator) => {
|
||||
await mutator.commit(writes, BACKGROUND_CONTEXT);
|
||||
}, BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
function address(write: Write): string {
|
||||
if (write.kind === "entry") return `entry:${write.entry.id}`;
|
||||
if (write.kind === "usage") return `usage:${write.row.id}`;
|
||||
return `${write.kind}:${write.op}:${write.namespace}:${write.key}`;
|
||||
}
|
||||
|
||||
async function seedLeftovers(session: Session, operationId: string, state: OperationState): Promise<void> {
|
||||
await commit(session, [
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta(operationId, state)),
|
||||
storedValues.setValue(storedValues.operationState(operationId), state),
|
||||
storedValues.setValue(storedValues.operationToolArgs(operationId, "step", 0), { value: true }),
|
||||
storedValues.setValue(storedValues.operationToolMemo(operationId, "invocation", "memo"), { value: true }),
|
||||
storedValues.setValue(storedValues.operationPreparation(operationId, "task"), {
|
||||
kind: "branch_summary",
|
||||
messages: [],
|
||||
fileOps: { read: [], written: [], edited: [] },
|
||||
totalTokens: 0,
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingToolOutput(operationId, "invocation"), {
|
||||
content: [{ type: "text", text: "partial" }],
|
||||
details: {},
|
||||
}),
|
||||
]);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
vi.restoreAllMocks();
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime terminal cleanup mechanics", () => {
|
||||
it("deletes every operation-owned family and exact live frame list but preserves the lane inbox", async () => {
|
||||
const { session } = await createSession();
|
||||
const operationId = "run";
|
||||
const responseEntryId = "response";
|
||||
const state: AssistantEffectPendingOperation = {
|
||||
...runScope(),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: {
|
||||
stepId: "step",
|
||||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
responseEntryId,
|
||||
usageId: "usage",
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1_000,
|
||||
control: { status: "cancel_requested", requestedAt: 2 },
|
||||
};
|
||||
await seedLeftovers(session, operationId, state);
|
||||
await commit(session, [
|
||||
...["steer", "follow", "write", "next"].map((id) =>
|
||||
storedValues.setValue(storedValues.pendingEntry(id), {
|
||||
type: "custom",
|
||||
customType: "test",
|
||||
payload: { id },
|
||||
}),
|
||||
),
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(operationId, responseEntryId), {
|
||||
type: "text_delta",
|
||||
contentIndex: 0,
|
||||
delta: "partial",
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [
|
||||
{ entryId: "steer", kind: "steer" },
|
||||
{ entryId: "follow", kind: "followUp" },
|
||||
{ entryId: "write", kind: "write" },
|
||||
{ entryId: "next", kind: "nextRun" },
|
||||
],
|
||||
}),
|
||||
]);
|
||||
|
||||
const writes = await operationCleanupWrites(session, operationId, state, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(writes.map(address)).toEqual([
|
||||
"value:delete:pi.op.meta:run",
|
||||
"value:delete:pi.op.state:run",
|
||||
"value:delete:pi.op.tool_args:run:step:0",
|
||||
"value:delete:pi.op.tool_memo:run:invocation:memo",
|
||||
"value:delete:pi.op.preparation:run:task",
|
||||
"value:delete:pi.pending.tool_output:run:invocation",
|
||||
"list:delete:pi.pending.assistant_frame:run:response",
|
||||
]);
|
||||
await commit(session, writes);
|
||||
for (const id of ["steer", "follow", "write", "next"]) {
|
||||
expect(await session.getValue(storedValues.pendingEntry(id), BACKGROUND_CONTEXT)).toBeDefined();
|
||||
}
|
||||
expect((await session.getValue(storedValues.laneState("main"), BACKGROUND_CONTEXT))?.value.inbox).toEqual([
|
||||
{ entryId: "steer", kind: "steer" },
|
||||
{ entryId: "follow", kind: "followUp" },
|
||||
{ entryId: "write", kind: "write" },
|
||||
{ entryId: "next", kind: "nextRun" },
|
||||
]);
|
||||
expect(
|
||||
await session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
});
|
||||
|
||||
it("deletes staged tool outcomes and leaves completed results alone", async () => {
|
||||
const { session } = await createSession();
|
||||
const operationId = "tools";
|
||||
const state: ToolsOperation = {
|
||||
...runScope(),
|
||||
at: "tools",
|
||||
batch: {
|
||||
assistantEntryId: "assistant",
|
||||
configuration,
|
||||
turnId: "step",
|
||||
calls: [
|
||||
{ status: "outcome_ready", sourceIndex: 0, resultEntryId: "staged", terminate: false },
|
||||
{ status: "completed", sourceIndex: 1, resultEntryId: "placed", terminate: false },
|
||||
],
|
||||
},
|
||||
};
|
||||
await seedLeftovers(session, operationId, state);
|
||||
await commit(session, [
|
||||
storedValues.setValue(storedValues.pendingEntry("staged"), {
|
||||
type: "message",
|
||||
payload: {
|
||||
role: "toolResult",
|
||||
toolCallId: "call",
|
||||
toolName: "tool",
|
||||
content: [],
|
||||
isError: false,
|
||||
timestamp: 1,
|
||||
},
|
||||
}),
|
||||
]);
|
||||
|
||||
const writes = await operationCleanupWrites(session, operationId, state, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(writes.map(address)).toContain("value:delete:pi.pending.entry:staged");
|
||||
expect(writes.map(address)).not.toContain("value:delete:pi.pending.entry:placed");
|
||||
});
|
||||
|
||||
it.each([
|
||||
[
|
||||
"compaction",
|
||||
{
|
||||
...runScope(),
|
||||
at: "summary.deciding",
|
||||
task: { taskId: "task", reason: "manual", boundary: { kind: "finish" } },
|
||||
} satisfies SummaryDecidingOperation,
|
||||
],
|
||||
[
|
||||
"navigation",
|
||||
{
|
||||
...runScope(),
|
||||
at: "navigation.ready_to_commit",
|
||||
targetId: null,
|
||||
} satisfies NavigationReadyToCommitOperation,
|
||||
],
|
||||
] as const)("defensively deletes leftover %s operation families", async (operationId, state) => {
|
||||
const { session } = await createSession();
|
||||
await seedLeftovers(session, operationId, state);
|
||||
|
||||
const writes = await operationCleanupWrites(session, operationId, state, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(writes.map(address)).toEqual([
|
||||
`value:delete:pi.op.meta:${operationId}`,
|
||||
`value:delete:pi.op.state:${operationId}`,
|
||||
`value:delete:pi.op.tool_args:${operationId}:step:0`,
|
||||
`value:delete:pi.op.tool_memo:${operationId}:invocation:memo`,
|
||||
`value:delete:pi.op.preparation:${operationId}:task`,
|
||||
`value:delete:pi.pending.tool_output:${operationId}:invocation`,
|
||||
]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime operation result records", () => {
|
||||
it("constructs one flat immutable observation from terminal metadata", () => {
|
||||
vi.spyOn(Date, "now").mockReturnValue(20);
|
||||
const record = operationResultRecord(
|
||||
{
|
||||
operationId: "run",
|
||||
lane: "main",
|
||||
sourceTipId: "source",
|
||||
startedAt: 10,
|
||||
intent: { kind: "run", promptEntryIds: ["prompt"] },
|
||||
},
|
||||
"failed",
|
||||
"tip",
|
||||
{ code: "provider", message: "failed" },
|
||||
);
|
||||
|
||||
expect(record).toEqual({
|
||||
operationId: "run",
|
||||
kind: "run",
|
||||
status: "failed",
|
||||
error: { code: "provider", message: "failed" },
|
||||
fromTipId: "source",
|
||||
tipId: "tip",
|
||||
startedAt: 10,
|
||||
endedAt: 20,
|
||||
});
|
||||
});
|
||||
|
||||
it("rejects errors on non-failed records and missing errors on failed records", () => {
|
||||
const metadata: OperationMeta = {
|
||||
operationId: "run",
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
};
|
||||
expect(() => operationResultRecord(metadata, "completed", "tip", { code: "x", message: "x" })).toThrow(
|
||||
"Only a failed operation result may carry an error",
|
||||
);
|
||||
expect(() => operationResultRecord(metadata, "failed", "tip")).toThrow(
|
||||
"Only a failed operation result may carry an error",
|
||||
);
|
||||
});
|
||||
});
|
||||
763
third_party/pi/packages/agent/test/harness/runtime/drive-tools.test.ts
vendored
Normal file
763
third_party/pi/packages/agent/test/harness/runtime/drive-tools.test.ts
vendored
Normal file
@@ -0,0 +1,763 @@
|
||||
import {
|
||||
createModels,
|
||||
fauxAssistantMessage,
|
||||
fauxProvider,
|
||||
fauxToolCall,
|
||||
type ToolResultMessage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { HarnessEvent, WatchHandle } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT, type Context } from "../../../src/harness/context.ts";
|
||||
import { HookRegistry } from "../../../src/harness/hooks.ts";
|
||||
import { runTools } from "../../../src/harness/runtime/drive/tools.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { restoreLane } from "../../../src/harness/runtime/restore.ts";
|
||||
import { type Config, Drive } from "../../../src/harness/runtime/types.ts";
|
||||
import { insertEntry } from "../../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type {
|
||||
LaneConfiguration,
|
||||
OperationState,
|
||||
Session,
|
||||
ToolCall,
|
||||
ToolsOperation,
|
||||
Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import type {
|
||||
AgentHarnessTool,
|
||||
AgentHarnessToolInvocation,
|
||||
AgentHarnessToolUpdateCallback,
|
||||
} from "../../../src/harness/types.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const schema = Type.Object({ value: Type.String() });
|
||||
|
||||
interface Fixture {
|
||||
session: Session;
|
||||
lane: Lane<undefined>;
|
||||
drive: Drive;
|
||||
hooks: HookRegistry;
|
||||
events: HarnessEvent[];
|
||||
assistantEntryId: string;
|
||||
resultEntryIds: string[];
|
||||
operationId: string;
|
||||
observations: string[];
|
||||
}
|
||||
|
||||
interface FixtureOptions {
|
||||
calls: Array<{ name: string; value?: string }>;
|
||||
tools?: AgentHarnessTool<undefined>[];
|
||||
mode?: "sequential" | "parallel";
|
||||
stopReason?: "toolUse" | "length";
|
||||
callStates?: (resultEntryIds: string[]) => ToolCall[];
|
||||
extraWrites?: (fixture: {
|
||||
operationId: string;
|
||||
assistantEntryId: string;
|
||||
resultEntryIds: string[];
|
||||
run: ToolsOperation;
|
||||
}) => Write[];
|
||||
toolContext?: Config<undefined>["toolContext"];
|
||||
cancelled?: boolean;
|
||||
onEmit?: (events: readonly HarnessEvent[]) => Promise<void>;
|
||||
}
|
||||
|
||||
function unusedWatch<T>(): WatchHandle<T> {
|
||||
throw new Error("watch is not used by tool procedure tests");
|
||||
}
|
||||
|
||||
class ObservedMemoryStorage extends MemoryStorage {
|
||||
readonly observations: string[] = [];
|
||||
|
||||
override async commit(writes: Write[], context: Context) {
|
||||
const result = await super.commit(writes, context);
|
||||
if (
|
||||
writes.some((write) => write.kind === "value" && write.op === "set" && write.namespace === "pi.op.tool_args")
|
||||
) {
|
||||
this.observations.push("intent_commit");
|
||||
}
|
||||
const stagesOutcome = writes.some(
|
||||
(write) => write.kind === "value" && write.op === "set" && write.namespace === "pi.pending.entry",
|
||||
);
|
||||
if (stagesOutcome) this.observations.push("outcome_commit");
|
||||
if (
|
||||
!stagesOutcome &&
|
||||
writes.some(
|
||||
(write) => write.kind === "value" && write.op === "delete" && write.namespace === "pi.pending.tool_output",
|
||||
)
|
||||
) {
|
||||
this.observations.push("replay_commit");
|
||||
}
|
||||
return result;
|
||||
}
|
||||
}
|
||||
|
||||
function deferred<T = void>(): { promise: Promise<T>; resolve(value: T): void } {
|
||||
let resolvePromise: ((value: T) => void) | undefined;
|
||||
const promise = new Promise<T>((resolve) => {
|
||||
resolvePromise = resolve;
|
||||
});
|
||||
return { promise, resolve: (value) => resolvePromise?.(value) };
|
||||
}
|
||||
|
||||
async function waitFor(predicate: () => boolean): Promise<void> {
|
||||
for (let index = 0; index < 200; index += 1) {
|
||||
if (predicate()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
throw new Error("condition was not reached");
|
||||
}
|
||||
|
||||
function tool(
|
||||
name: string,
|
||||
execute: (
|
||||
value: string,
|
||||
onUpdate: AgentHarnessToolUpdateCallback<{ progress?: string }> | undefined,
|
||||
invocation: AgentHarnessToolInvocation,
|
||||
context: Parameters<AgentHarnessTool<undefined>["execute"]>[5],
|
||||
) => Promise<{
|
||||
content: Array<{ type: "text"; text: string }>;
|
||||
details: { value?: string };
|
||||
usage?: ToolResultMessage["usage"];
|
||||
addedToolNames?: string[];
|
||||
terminate?: boolean;
|
||||
}>,
|
||||
replay: "never" | "safe" = "never",
|
||||
): AgentHarnessTool<undefined, typeof schema, { progress?: string } | { value?: string }> {
|
||||
return {
|
||||
name,
|
||||
label: name,
|
||||
description: name,
|
||||
parameters: schema,
|
||||
replay,
|
||||
execute: async (_toolCallId, args, onUpdate, _toolContext, invocation, context) =>
|
||||
execute(args.value, onUpdate, invocation, context),
|
||||
};
|
||||
}
|
||||
|
||||
async function createFixture(options: FixtureOptions): Promise<Fixture> {
|
||||
const storage = new ObservedMemoryStorage({ now: () => 100 });
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `drive-tools-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
const operationId = session.idGenerator.next(10);
|
||||
const assistantEntryId = session.idGenerator.next(20);
|
||||
const resultEntryIds = options.calls.map(() => session.idGenerator.next(20));
|
||||
const faux = fauxProvider();
|
||||
const model = faux.getModel();
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: model.provider, modelId: model.id },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: options.calls.map(({ name }) => name),
|
||||
};
|
||||
const assistant = fauxAssistantMessage(
|
||||
options.calls.map(({ name, value = name }, index) => fauxToolCall(name, { value }, { id: `call-${index}` })),
|
||||
{ stopReason: options.stopReason ?? "toolUse", timestamp: 20 },
|
||||
);
|
||||
const calls =
|
||||
options.callStates?.(resultEntryIds) ??
|
||||
resultEntryIds.map((resultEntryId, index) => ({
|
||||
status: "planned" as const,
|
||||
sourceIndex: index,
|
||||
resultEntryId,
|
||||
}));
|
||||
const run: ToolsOperation = {
|
||||
at: "tools",
|
||||
control: options.cancelled ? { status: "cancel_requested", requestedAt: 30 } : { status: "running" },
|
||||
settings: {
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: options.mode ?? "parallel",
|
||||
},
|
||||
batch: { assistantEntryId, configuration, turnId: "turn-1", calls },
|
||||
latestAssistantEntryId: assistantEntryId,
|
||||
};
|
||||
const writes: Write[] = [
|
||||
insertEntry({ id: assistantEntryId, parentId: null, type: "message", message: assistant }),
|
||||
storedValues.setValue(storedValues.branchTip("main"), assistantEntryId),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 10,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationState(operationId), run),
|
||||
...(options.extraWrites?.({ operationId, assistantEntryId, resultEntryIds, run }) ?? []),
|
||||
];
|
||||
await session.mutate((mutator) => mutator.commit(writes, BACKGROUND_CONTEXT), BACKGROUND_CONTEXT);
|
||||
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
const hooks = new HookRegistry(() => {});
|
||||
const events: HarnessEvent[] = [];
|
||||
const config: Config<undefined> = {
|
||||
tools: options.tools ?? [],
|
||||
resources: {},
|
||||
streamOptions: {},
|
||||
retryPolicy: { enabled: false, maxRetries: 0, baseDelayMs: 0 },
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: options.mode ?? "parallel",
|
||||
toolContext: options.toolContext,
|
||||
systemPrompt: undefined,
|
||||
toProviderMessages: (messages) =>
|
||||
messages.filter(
|
||||
(message) => message.role === "user" || message.role === "assistant" || message.role === "toolResult",
|
||||
),
|
||||
entryProjectors: {},
|
||||
};
|
||||
const lane = new Lane<undefined>(
|
||||
"main",
|
||||
session,
|
||||
models,
|
||||
hooks,
|
||||
await restoreLane(session, "main", BACKGROUND_CONTEXT),
|
||||
(cause) => (cause instanceof Error ? cause : new Error(String(cause))),
|
||||
async (batch) => {
|
||||
events.push(...structuredClone(batch));
|
||||
storage.observations.push(...batch.map((event) => event.type));
|
||||
await options.onEmit?.(batch);
|
||||
},
|
||||
unusedWatch,
|
||||
() => config,
|
||||
);
|
||||
const drive = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
lane.activeDrive = drive;
|
||||
return {
|
||||
session,
|
||||
lane,
|
||||
drive,
|
||||
hooks,
|
||||
events,
|
||||
assistantEntryId,
|
||||
resultEntryIds,
|
||||
operationId,
|
||||
observations: storage.observations,
|
||||
};
|
||||
}
|
||||
|
||||
function currentRun(fixture: Fixture): OperationState {
|
||||
const operation = fixture.lane.state.operation;
|
||||
if (operation === null) throw new Error("fixture has no operation");
|
||||
return operation.state;
|
||||
}
|
||||
|
||||
function currentCalls(fixture: Fixture): ToolCall[] {
|
||||
const run = currentRun(fixture);
|
||||
if (run.at !== "tools") return [];
|
||||
return run.batch.calls;
|
||||
}
|
||||
|
||||
async function driveTools(fixture: Fixture) {
|
||||
const run = currentRun(fixture);
|
||||
if (run.at !== "tools") throw new Error("fixture has no tool batch");
|
||||
return runTools(fixture.lane, fixture.drive, run);
|
||||
}
|
||||
|
||||
async function expectProjectionRestores(fixture: Fixture): Promise<void> {
|
||||
expect(await restoreLane(fixture.session, "main", BACKGROUND_CONTEXT)).toEqual(fixture.lane.state);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
await Promise.all(sessions.splice(0).map((session) => session.close(BACKGROUND_CONTEXT)));
|
||||
});
|
||||
|
||||
describe("durable tool batch", () => {
|
||||
it("executes a sequential batch with memos, checkpoints, hooks, usage, and source-order placement", async () => {
|
||||
let contextResolutions = 0;
|
||||
let lateInvocation: AgentHarnessToolInvocation | undefined;
|
||||
const usage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 3,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
const first = tool("first", async (value, onUpdate, invocation) => {
|
||||
lateInvocation = invocation;
|
||||
await invocation.setMemo("step/a", { value: "memo" });
|
||||
expect(await invocation.getMemo("step/a")).toEqual({ value: "memo" });
|
||||
onUpdate?.(
|
||||
{ content: [{ type: "text", text: "partial" }], details: { progress: "partial" } },
|
||||
{ checkpoint: true },
|
||||
);
|
||||
return { content: [{ type: "text", text: value }], details: { value }, usage };
|
||||
});
|
||||
const second = tool("second", async (value) => ({
|
||||
content: [{ type: "text", text: value }],
|
||||
details: { value },
|
||||
addedToolNames: ["introduced"],
|
||||
}));
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "first" }, { name: "second" }],
|
||||
tools: [first, second],
|
||||
mode: "sequential",
|
||||
toolContext: () => {
|
||||
contextResolutions += 1;
|
||||
return undefined;
|
||||
},
|
||||
});
|
||||
let checkpointSeenByAfterHook = false;
|
||||
fixture.hooks.on("before_tool", ({ toolName, args }) =>
|
||||
toolName === "first" ? { args: { ...args, value: "prepared" } } : undefined,
|
||||
);
|
||||
fixture.hooks.on("after_tool", async ({ toolName }) => {
|
||||
if (toolName === "first") {
|
||||
checkpointSeenByAfterHook =
|
||||
(await fixture.session.getValue(
|
||||
storedValues.pendingToolOutput(fixture.operationId, fixture.resultEntryIds[0]!),
|
||||
BACKGROUND_CONTEXT,
|
||||
)) !== undefined;
|
||||
}
|
||||
return undefined;
|
||||
});
|
||||
|
||||
expect(await driveTools(fixture)).toEqual({ kind: "continue" });
|
||||
const transcript = await fixture.lane.findEntries({ order: "oldestFirst" }, BACKGROUND_CONTEXT);
|
||||
expect(transcript.map(({ id }) => id)).toEqual([fixture.assistantEntryId, ...fixture.resultEntryIds]);
|
||||
expect(transcript[1]?.type === "message" ? transcript[1].message : undefined).toMatchObject({
|
||||
role: "toolResult",
|
||||
content: [{ type: "text", text: "prepared" }],
|
||||
});
|
||||
expect(contextResolutions).toBe(1);
|
||||
expect(checkpointSeenByAfterHook).toBe(true);
|
||||
expect(currentRun(fixture)).toMatchObject({
|
||||
at: "checkpoint",
|
||||
continuation: { kind: "need_assistant" },
|
||||
});
|
||||
expect(fixture.lane.state.configuration.activeToolNames).toEqual(["first", "second", "introduced"]);
|
||||
expect(
|
||||
await fixture.session.scanValues(
|
||||
storedValues.operationToolArgsPrefix(fixture.operationId),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await fixture.session.scanValues(
|
||||
storedValues.operationToolMemoPrefix(fixture.operationId),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(
|
||||
await fixture.session.getValue(
|
||||
storedValues.pendingToolOutput(fixture.operationId, fixture.resultEntryIds[0]!),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toBeUndefined();
|
||||
await expect(lateInvocation?.setMemo("late", true)).rejects.toThrow("no longer owns");
|
||||
const eventTypes = fixture.events.map(({ type }) => type);
|
||||
expect(eventTypes.indexOf("tool_update")).toBeLessThan(eventTypes.indexOf("tool_end"));
|
||||
const firstIntent = fixture.observations.indexOf("intent_commit");
|
||||
const firstStart = fixture.observations.indexOf("tool_start");
|
||||
const firstUpdate = fixture.observations.indexOf("tool_update");
|
||||
const firstCommit = fixture.observations.indexOf("outcome_commit");
|
||||
const firstEnd = fixture.observations.indexOf("tool_end");
|
||||
const firstPlacement = fixture.observations.indexOf("entry_added");
|
||||
expect(firstIntent).toBeLessThan(firstStart);
|
||||
expect(firstStart).toBeLessThan(firstUpdate);
|
||||
expect(firstUpdate).toBeLessThan(firstCommit);
|
||||
expect(firstCommit).toBeLessThan(firstEnd);
|
||||
expect(firstEnd).toBeLessThan(firstPlacement);
|
||||
expect(fixture.events.filter(({ type }) => type === "entry_added")).toHaveLength(2);
|
||||
expect(fixture.events.filter(({ type }) => type === "usage")).toHaveLength(1);
|
||||
expect(fixture.events.at(-1)?.type).toBe("turn_end");
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("stages parallel completion order but materializes source order", async () => {
|
||||
const finishA = deferred<void>();
|
||||
const finishB = deferred<void>();
|
||||
const started: string[] = [];
|
||||
const make = (name: string, finish: ReturnType<typeof deferred<void>>) =>
|
||||
tool(name, async () => {
|
||||
started.push(name);
|
||||
await finish.promise;
|
||||
return { content: [{ type: "text", text: name }], details: { value: name } };
|
||||
});
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "a" }, { name: "b" }],
|
||||
tools: [make("a", finishA), make("b", finishB)],
|
||||
mode: "parallel",
|
||||
});
|
||||
const running = driveTools(fixture);
|
||||
await waitFor(() => started.length === 2);
|
||||
finishB.resolve();
|
||||
await waitFor(() => currentCalls(fixture)[1]?.status === "outcome_ready");
|
||||
expect(currentCalls(fixture).map(({ status }) => status)).toEqual(["effect_pending", "outcome_ready"]);
|
||||
expect(await fixture.session.getEntry(fixture.resultEntryIds[1]!, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(fixture.events.find((event) => event.type === "tool_start" && event.toolName === "b")).toMatchObject({
|
||||
args: { value: "b" },
|
||||
});
|
||||
expect(fixture.events.find((event) => event.type === "tool_end" && event.toolName === "b")).toMatchObject({
|
||||
result: { content: [{ type: "text", text: "b" }] },
|
||||
});
|
||||
finishA.resolve();
|
||||
await running;
|
||||
|
||||
const transcript = await fixture.lane.findEntries({ order: "oldestFirst" }, BACKGROUND_CONTEXT);
|
||||
expect(transcript.map(({ id }) => id)).toEqual([
|
||||
fixture.assistantEntryId,
|
||||
fixture.resultEntryIds[0],
|
||||
fixture.resultEntryIds[1],
|
||||
]);
|
||||
await expectProjectionRestores(fixture);
|
||||
});
|
||||
|
||||
it("safe-replays persisted arguments and memos while interrupting unsafe effects", async () => {
|
||||
const safeExecute = vi.fn(async (_value: string, _onUpdate: unknown, invocation: AgentHarnessToolInvocation) => {
|
||||
expect(invocation.invocationId).toBeDefined();
|
||||
expect(await invocation.getMemo("step/a")).toEqual({ complete: true });
|
||||
return { content: [{ type: "text" as const, text: "safe replay" }], details: { value: "safe" } };
|
||||
});
|
||||
const unsafeExecute = vi.fn(async () => ({
|
||||
content: [{ type: "text" as const, text: "must not run" }],
|
||||
details: {},
|
||||
}));
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "safe" }, { name: "unsafe" }],
|
||||
tools: [tool("safe", safeExecute, "safe"), tool("unsafe", unsafeExecute, "never")],
|
||||
callStates: (ids) => [
|
||||
{ status: "effect_pending", sourceIndex: 0, resultEntryId: ids[0]!, replay: "safe" },
|
||||
{ status: "effect_pending", sourceIndex: 1, resultEntryId: ids[1]!, replay: "never" },
|
||||
],
|
||||
extraWrites: ({ operationId, resultEntryIds }) => [
|
||||
storedValues.setValue(storedValues.operationToolArgs(operationId, "turn-1", 0), { value: "persisted" }),
|
||||
storedValues.setValue(storedValues.operationToolArgs(operationId, "turn-1", 1), { value: "unsafe" }),
|
||||
storedValues.setValue(storedValues.operationToolMemo(operationId, resultEntryIds[0]!, "step/a"), {
|
||||
complete: true,
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingToolOutput(operationId, resultEntryIds[0]!), {
|
||||
content: [{ type: "text", text: "old progress" }],
|
||||
details: {},
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingToolOutput(operationId, resultEntryIds[1]!), {
|
||||
content: [{ type: "text", text: "durable partial" }],
|
||||
details: { progress: "kept" },
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
await driveTools(fixture);
|
||||
expect(safeExecute).toHaveBeenCalledOnce();
|
||||
expect(safeExecute.mock.calls[0]?.[0]).toBe("persisted");
|
||||
expect(unsafeExecute).not.toHaveBeenCalled();
|
||||
expect(
|
||||
fixture.events.find((event) => event.type === "tool_start" && event.toolCallId === "call-0"),
|
||||
).toMatchObject({ recovery: true, args: { value: "persisted" } });
|
||||
expect(fixture.observations.indexOf("replay_commit")).toBeLessThan(fixture.observations.indexOf("tool_start"));
|
||||
expect(fixture.events.find((event) => event.type === "tool_end" && event.toolCallId === "call-0")).toMatchObject({
|
||||
recovery: true,
|
||||
isError: false,
|
||||
});
|
||||
const unsafeEntry = await fixture.session.getEntry(fixture.resultEntryIds[1]!, BACKGROUND_CONTEXT);
|
||||
expect(unsafeEntry?.type === "message" ? unsafeEntry.message : undefined).toMatchObject({
|
||||
role: "toolResult",
|
||||
isError: true,
|
||||
details: { progress: "kept" },
|
||||
});
|
||||
expect(
|
||||
unsafeEntry?.type === "message" && unsafeEntry.message.role === "toolResult"
|
||||
? unsafeEntry.message.content.at(-1)
|
||||
: undefined,
|
||||
).toMatchObject({ type: "text", text: expect.stringContaining("external outcome is unknown") });
|
||||
expect(fixture.events.some((event) => event.type === "tool_start" && event.toolCallId === "call-1")).toBe(false);
|
||||
expect(fixture.events.find((event) => event.type === "tool_end" && event.toolCallId === "call-1")).toMatchObject({
|
||||
recovery: true,
|
||||
isError: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("reconciles a restored cancelled batch without hooks, context, or effects", async () => {
|
||||
const execute = vi.fn(async () => ({ content: [], details: {} }));
|
||||
const context = vi.fn(() => undefined);
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "planned" }, { name: "pending" }],
|
||||
tools: [tool("planned", execute), tool("pending", execute, "safe")],
|
||||
toolContext: context,
|
||||
cancelled: true,
|
||||
callStates: (ids) => [
|
||||
{ status: "planned", sourceIndex: 0, resultEntryId: ids[0]! },
|
||||
{ status: "effect_pending", sourceIndex: 1, resultEntryId: ids[1]!, replay: "safe" },
|
||||
],
|
||||
extraWrites: ({ operationId, resultEntryIds }) => [
|
||||
storedValues.setValue(storedValues.operationToolArgs(operationId, "turn-1", 1), { value: "pending" }),
|
||||
storedValues.setValue(storedValues.pendingToolOutput(operationId, resultEntryIds[1]!), {
|
||||
content: [{ type: "text", text: "checkpoint" }],
|
||||
details: {},
|
||||
}),
|
||||
],
|
||||
});
|
||||
const beforeTool = vi.fn(() => undefined);
|
||||
const afterTool = vi.fn(() => undefined);
|
||||
fixture.hooks.on("before_tool", beforeTool);
|
||||
fixture.hooks.on("after_tool", afterTool);
|
||||
|
||||
await driveTools(fixture);
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
expect(context).not.toHaveBeenCalled();
|
||||
expect(beforeTool).not.toHaveBeenCalled();
|
||||
expect(afterTool).not.toHaveBeenCalled();
|
||||
expect(currentRun(fixture)).toMatchObject({
|
||||
at: "checkpoint",
|
||||
control: { status: "cancel_requested" },
|
||||
continuation: { kind: "need_assistant" },
|
||||
});
|
||||
const pending = await fixture.session.getEntry(fixture.resultEntryIds[1]!, BACKGROUND_CONTEXT);
|
||||
expect(
|
||||
pending?.type === "message" && pending.message.role === "toolResult"
|
||||
? pending.message.content.at(-1)
|
||||
: undefined,
|
||||
).toMatchObject({ type: "text", text: expect.stringContaining("external outcome is unknown") });
|
||||
const starts = fixture.events.filter((event) => event.type === "tool_start");
|
||||
const ends = fixture.events.filter((event) => event.type === "tool_end");
|
||||
expect(starts).toHaveLength(1);
|
||||
expect(starts[0]).toMatchObject({ toolName: "planned", args: { value: "planned" } });
|
||||
expect(ends).toHaveLength(2);
|
||||
expect(ends.find((event) => event.toolCallId === starts[0]?.toolCallId)).toMatchObject({ isError: true });
|
||||
});
|
||||
|
||||
it("materializes outcome-ready state without resolving tools or tool context", async () => {
|
||||
const execute = vi.fn(async () => ({ content: [], details: {} }));
|
||||
const context = vi.fn(() => undefined);
|
||||
const staged: ToolResultMessage = {
|
||||
role: "toolResult",
|
||||
toolCallId: "call-0",
|
||||
toolName: "ready",
|
||||
content: [{ type: "text", text: "already done" }],
|
||||
addedToolNames: ["later"],
|
||||
isError: false,
|
||||
timestamp: 30,
|
||||
};
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "ready" }],
|
||||
tools: [tool("ready", execute)],
|
||||
toolContext: context,
|
||||
callStates: (ids) => [{ status: "outcome_ready", sourceIndex: 0, resultEntryId: ids[0]!, terminate: true }],
|
||||
extraWrites: ({ resultEntryIds }) => [
|
||||
storedValues.setValue(storedValues.pendingEntry(resultEntryIds[0]!), {
|
||||
type: "message",
|
||||
payload: staged,
|
||||
}),
|
||||
],
|
||||
});
|
||||
|
||||
await driveTools(fixture);
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
expect(context).not.toHaveBeenCalled();
|
||||
expect(currentRun(fixture)).toMatchObject({
|
||||
at: "checkpoint",
|
||||
continuation: { kind: "may_finish", includeFinalAssistant: false },
|
||||
});
|
||||
expect(fixture.lane.state.configuration.activeToolNames).toEqual(["ready", "later"]);
|
||||
expect(fixture.events[0]).toMatchObject({ type: "turn_start", turnId: "turn-1", recovery: true });
|
||||
expect(fixture.events.at(-1)).toMatchObject({ type: "turn_end", turnId: "turn-1", recovery: true });
|
||||
});
|
||||
|
||||
it("never executes genuine-length or missing tool calls", async () => {
|
||||
const execute = vi.fn(async () => ({ content: [], details: {} }));
|
||||
const truncated = await createFixture({
|
||||
calls: [{ name: "present" }],
|
||||
tools: [tool("present", execute)],
|
||||
stopReason: "length",
|
||||
});
|
||||
const truncatedAfterTool = vi.fn(() => undefined);
|
||||
truncated.hooks.on("after_tool", truncatedAfterTool);
|
||||
await driveTools(truncated);
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
const truncatedEntry = await truncated.session.getEntry(truncated.resultEntryIds[0]!, BACKGROUND_CONTEXT);
|
||||
expect(
|
||||
truncatedEntry?.type === "message" && truncatedEntry.message.role === "toolResult"
|
||||
? truncatedEntry.message.content[0]
|
||||
: undefined,
|
||||
).toMatchObject({ type: "text", text: expect.stringContaining("arguments may be truncated") });
|
||||
expect(truncatedAfterTool).not.toHaveBeenCalled();
|
||||
expect(truncated.events.filter((event) => event.type === "tool_start")).toHaveLength(1);
|
||||
expect(truncated.events.filter((event) => event.type === "tool_end")).toHaveLength(1);
|
||||
expect(truncated.observations.indexOf("outcome_commit")).toBeLessThan(
|
||||
truncated.observations.indexOf("tool_start"),
|
||||
);
|
||||
expect(truncated.observations.indexOf("tool_start")).toBeLessThan(truncated.observations.indexOf("tool_end"));
|
||||
expect(truncated.observations.indexOf("tool_end")).toBeLessThan(truncated.observations.indexOf("entry_added"));
|
||||
|
||||
const missing = await createFixture({ calls: [{ name: "missing" }], tools: [] });
|
||||
const missingAfterTool = vi.fn(() => undefined);
|
||||
missing.hooks.on("after_tool", missingAfterTool);
|
||||
await driveTools(missing);
|
||||
const missingEntry = await missing.session.getEntry(missing.resultEntryIds[0]!, BACKGROUND_CONTEXT);
|
||||
expect(missingEntry?.type === "message" ? missingEntry.message : undefined).toMatchObject({
|
||||
role: "toolResult",
|
||||
isError: true,
|
||||
content: [{ type: "text", text: 'Tool "missing" is unavailable' }],
|
||||
});
|
||||
expect(
|
||||
missingEntry?.type === "message" && missingEntry.message.role === "toolResult"
|
||||
? Object.hasOwn(missingEntry.message, "details")
|
||||
: true,
|
||||
).toBe(false);
|
||||
expect(missingAfterTool).not.toHaveBeenCalled();
|
||||
expect(missing.events.filter((event) => event.type === "tool_start")).toHaveLength(1);
|
||||
const missingEnds = missing.events.filter((event) => event.type === "tool_end");
|
||||
expect(missingEnds).toHaveLength(1);
|
||||
expect(missing.events.find((event) => event.type === "tool_start")).toMatchObject({
|
||||
args: { value: "missing" },
|
||||
});
|
||||
expect(missingEnds[0]).toMatchObject({
|
||||
result: { content: [{ type: "text", text: 'Tool "missing" is unavailable' }], details: undefined },
|
||||
isError: true,
|
||||
});
|
||||
});
|
||||
|
||||
it("awaits update delivery and checkpoint persistence before after_tool", async () => {
|
||||
const releaseUpdate = deferred<void>();
|
||||
const updateQueued = deferred<void>();
|
||||
let afterStarted = false;
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "updating" }],
|
||||
tools: [
|
||||
tool("updating", async (_value, onUpdate) => {
|
||||
onUpdate?.(
|
||||
{ content: [{ type: "text", text: "partial" }], details: { progress: "partial" } },
|
||||
{ checkpoint: true },
|
||||
);
|
||||
return { content: [{ type: "text", text: "done" }], details: {} };
|
||||
}),
|
||||
],
|
||||
onEmit: async (events) => {
|
||||
if (events.some(({ type }) => type === "tool_update")) {
|
||||
updateQueued.resolve();
|
||||
await releaseUpdate.promise;
|
||||
}
|
||||
},
|
||||
});
|
||||
fixture.hooks.on("after_tool", async () => {
|
||||
afterStarted = true;
|
||||
expect(
|
||||
await fixture.session.getValue(
|
||||
storedValues.pendingToolOutput(fixture.operationId, fixture.resultEntryIds[0]!),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toBeDefined();
|
||||
return undefined;
|
||||
});
|
||||
|
||||
const running = driveTools(fixture);
|
||||
await updateQueued.promise;
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(afterStarted).toBe(false);
|
||||
releaseUpdate.resolve();
|
||||
await running;
|
||||
expect(afterStarted).toBe(true);
|
||||
});
|
||||
|
||||
it("does not stage a cancelled outcome before cancellation is durable", async () => {
|
||||
const execute = vi.fn(async () => ({ content: [], details: {} }));
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "cancel-before-admission" }],
|
||||
tools: [tool("cancel-before-admission", execute)],
|
||||
mode: "sequential",
|
||||
});
|
||||
const cancellation = deferred<void>();
|
||||
fixture.drive.beginAbort(cancellation.promise);
|
||||
const running = driveTools(fixture);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
expect(currentCalls(fixture)[0]?.status).toBe("planned");
|
||||
expect(
|
||||
await fixture.session.getValue(storedValues.pendingEntry(fixture.resultEntryIds[0]!), BACKGROUND_CONTEXT),
|
||||
).toBeUndefined();
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const nextRun: OperationState = {
|
||||
...operation.state,
|
||||
control: { status: "cancel_requested", requestedAt: 40 },
|
||||
};
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(fixture.operationId), nextRun)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: nextRun } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
cancellation.resolve();
|
||||
fixture.drive.signalAbort();
|
||||
await running;
|
||||
|
||||
expect(execute).not.toHaveBeenCalled();
|
||||
expect(await fixture.session.getEntry(fixture.resultEntryIds[0]!, BACKGROUND_CONTEXT)).toBeDefined();
|
||||
});
|
||||
|
||||
it("drains live updates before after_tool and stages a non-terminating result after cancellation", async () => {
|
||||
const updateDelivery = deferred<void>();
|
||||
const started = deferred<void>();
|
||||
let updateDelivered = false;
|
||||
const afterTool = vi.fn(() => {
|
||||
expect(updateDelivered).toBe(true);
|
||||
return undefined;
|
||||
});
|
||||
const executing = tool("slow", async (_value, onUpdate, _invocation, context) => {
|
||||
onUpdate?.({ content: [{ type: "text", text: "partial" }], details: { progress: "partial" } });
|
||||
started.resolve();
|
||||
await new Promise<void>((_resolve, reject) => {
|
||||
context.abortSignal?.addEventListener("abort", () => reject(new Error("cancelled effect")), { once: true });
|
||||
});
|
||||
return { content: [], details: {}, terminate: true };
|
||||
});
|
||||
const fixture = await createFixture({
|
||||
calls: [{ name: "slow" }],
|
||||
tools: [executing],
|
||||
mode: "sequential",
|
||||
onEmit: async (events) => {
|
||||
if (events.some(({ type }) => type === "tool_update")) {
|
||||
await updateDelivery.promise;
|
||||
updateDelivered = true;
|
||||
}
|
||||
},
|
||||
});
|
||||
fixture.hooks.on("after_tool", afterTool);
|
||||
const running = driveTools(fixture);
|
||||
await started.promise;
|
||||
const cancellation = deferred<void>();
|
||||
fixture.drive.beginAbort(cancellation.promise);
|
||||
await fixture.lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const nextRun: OperationState = {
|
||||
...operation.state,
|
||||
control: { status: "cancel_requested", requestedAt: 40 },
|
||||
};
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(fixture.operationId), nextRun)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: nextRun } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
cancellation.resolve();
|
||||
fixture.drive.signalAbort();
|
||||
updateDelivery.resolve();
|
||||
await running;
|
||||
expect(afterTool).not.toHaveBeenCalled();
|
||||
const entry = await fixture.session.getEntry(fixture.resultEntryIds[0]!, BACKGROUND_CONTEXT);
|
||||
expect(entry?.type === "message" ? entry.terminate : undefined).toBeUndefined();
|
||||
expect(entry?.type === "message" ? entry.message : undefined).toMatchObject({
|
||||
role: "toolResult",
|
||||
isError: true,
|
||||
});
|
||||
});
|
||||
});
|
||||
397
third_party/pi/packages/agent/test/harness/runtime/harness.test.ts
vendored
Normal file
397
third_party/pi/packages/agent/test/harness/runtime/harness.test.ts
vendored
Normal file
@@ -0,0 +1,397 @@
|
||||
import { createModels, fauxAssistantMessage, fauxProvider, type Provider } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AgentHarness, HarnessFault, InvalidLane, UnknownTarget } from "../../../src/harness/agent-harness.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { Harness } from "../../../src/harness/runtime/harness.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { MemorySessionRepo, MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type { LaneConfiguration, Session } from "../../../src/harness/session/types.ts";
|
||||
import { branchTip, laneConfig, laneState, pendingEntry, setValue } from "../../../src/harness/session/values.ts";
|
||||
import { deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
|
||||
function harnessOptions(session: Session) {
|
||||
const provider = fauxProvider();
|
||||
const models = createModels();
|
||||
models.setProvider(provider.provider);
|
||||
return {
|
||||
session,
|
||||
models,
|
||||
model: provider.getModel(),
|
||||
thinkingLevel: "medium" as const,
|
||||
activeToolNames: ["read", "bash"],
|
||||
};
|
||||
}
|
||||
|
||||
async function createSession(id = `session-${sessions.length}`): Promise<Session> {
|
||||
const session = new StorageBackedSession({ id, createdAt: 1, storageVersion: 1 }, new MemoryStorage());
|
||||
sessions.push(session);
|
||||
return session;
|
||||
}
|
||||
|
||||
async function createHarness(session?: Session): Promise<Harness<object | undefined>> {
|
||||
session ??= await createSession();
|
||||
const created = await AgentHarness.create(harnessOptions(session), BACKGROUND_CONTEXT);
|
||||
if (!(created.harness instanceof Harness)) throw new Error("Expected runtime Harness");
|
||||
return created.harness;
|
||||
}
|
||||
|
||||
const configured = {
|
||||
model: { provider: "faux", modelId: "faux-1" },
|
||||
thinkingLevel: "low",
|
||||
activeToolNames: ["read"],
|
||||
} satisfies LaneConfiguration;
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime Harness lane management", () => {
|
||||
it("attaches to a fresh Session without creating an implicit main lane", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await createHarness(session);
|
||||
|
||||
expect(await harness.lanes(BACKGROUND_CONTEXT)).toEqual([]);
|
||||
expect(await session.branch("main", BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect("accept" in harness).toBe(false);
|
||||
expect("getTipId" in harness).toBe(false);
|
||||
expect("appendMessage" in harness).toBe(false);
|
||||
});
|
||||
|
||||
it("atomically gets or creates a complete AgentLane", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await createHarness(session);
|
||||
const events: string[] = [];
|
||||
harness.events.on("lane_created", (event) => {
|
||||
events.push(event.lane);
|
||||
});
|
||||
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
const same = await harness.lane("main", { createAt: "ignored" }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(lane).toBeInstanceOf(Lane);
|
||||
expect(same).toBe(lane);
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBeNull();
|
||||
expect(await lane.getThinkingLevel(BACKGROUND_CONTEXT)).toBe("medium");
|
||||
expect(await lane.getActiveTools(BACKGROUND_CONTEXT)).toEqual(["read", "bash"]);
|
||||
expect((await session.getValue(laneState("main"), BACKGROUND_CONTEXT))?.value).toEqual({
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
});
|
||||
expect(events).toEqual(["main"]);
|
||||
expect(await harness.lanes(BACKGROUND_CONTEXT)).toMatchObject([{ name: "main", tipId: null }]);
|
||||
});
|
||||
|
||||
it("returns one published AgentLane under concurrent acquisition", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await createHarness(session);
|
||||
const listener = vi.fn();
|
||||
harness.events.on("lane_created", listener);
|
||||
|
||||
const [first, second, third] = await Promise.all([
|
||||
harness.lane("main", BACKGROUND_CONTEXT),
|
||||
harness.lane("main", BACKGROUND_CONTEXT),
|
||||
harness.lane("main", BACKGROUND_CONTEXT),
|
||||
]);
|
||||
|
||||
expect(second).toBe(first);
|
||||
expect(third).toBe(first);
|
||||
expect(listener).toHaveBeenCalledTimes(1);
|
||||
});
|
||||
|
||||
it("uses one stable provider session id per lane", async () => {
|
||||
const session = await createSession("shared-session");
|
||||
const faux = fauxProvider();
|
||||
faux.setResponses([
|
||||
fauxAssistantMessage("main one"),
|
||||
fauxAssistantMessage("main two"),
|
||||
fauxAssistantMessage("review"),
|
||||
]);
|
||||
const sessionIds: Array<string | undefined> = [];
|
||||
const base = faux.provider;
|
||||
const provider: Provider = {
|
||||
id: base.id,
|
||||
name: base.name,
|
||||
auth: base.auth,
|
||||
getModels: () => base.getModels(),
|
||||
stream: (model, context, options) => base.stream(model, context, options),
|
||||
streamSimple: (model, context, options) => {
|
||||
sessionIds.push(options?.sessionId);
|
||||
return base.streamSimple(model, context, options);
|
||||
},
|
||||
};
|
||||
const models = createModels();
|
||||
models.setProvider(provider);
|
||||
const created = await AgentHarness.create(
|
||||
{ session, models, model: faux.getModel(), activeToolNames: [] },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const main = await created.harness.lane("main", BACKGROUND_CONTEXT);
|
||||
const review = await created.harness.lane("review", BACKGROUND_CONTEXT);
|
||||
|
||||
await main.prompt("one", undefined, BACKGROUND_CONTEXT);
|
||||
await main.prompt("two", undefined, BACKGROUND_CONTEXT);
|
||||
await review.prompt("review", undefined, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(sessionIds).toEqual(["shared-session:main", "shared-session:main", "shared-session:review"]);
|
||||
});
|
||||
|
||||
it("serializes commands from different AgentLanes on the one Session line", async () => {
|
||||
const harness = await createHarness();
|
||||
const main = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
const review = await harness.lane("review", BACKGROUND_CONTEXT);
|
||||
if (!(main instanceof Lane) || !(review instanceof Lane)) throw new Error("Expected runtime Lanes");
|
||||
const started = deferred();
|
||||
const gate = deferred();
|
||||
const order: string[] = [];
|
||||
const first = main.command(async () => {
|
||||
order.push("main:start");
|
||||
started.resolve();
|
||||
await gate.promise;
|
||||
order.push("main:end");
|
||||
return { kind: "return", result: undefined };
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await started.promise;
|
||||
const second = review.command(() => {
|
||||
order.push("review");
|
||||
return { kind: "return", result: undefined };
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await Promise.resolve();
|
||||
expect(order).toEqual(["main:start"]);
|
||||
gate.resolve();
|
||||
await Promise.all([first, second]);
|
||||
expect(order).toEqual(["main:start", "main:end", "review"]);
|
||||
});
|
||||
|
||||
it("uses createAt only for a missing lane and validates the target", async () => {
|
||||
const session = await createSession();
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "target",
|
||||
parentId: null,
|
||||
type: "custom",
|
||||
customType: "target",
|
||||
},
|
||||
},
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const harness = await createHarness(session);
|
||||
const lane = await harness.lane("review", { createAt: "target" }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBe("target");
|
||||
expect(await harness.lane("review", { createAt: "missing" }, BACKGROUND_CONTEXT)).toBe(lane);
|
||||
await expect(harness.lane("missing", { createAt: "unknown" }, BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(
|
||||
UnknownTarget,
|
||||
);
|
||||
await expect(harness.lane("", BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(InvalidLane);
|
||||
await expect(harness.lane("bad\u0000name", BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(InvalidLane);
|
||||
});
|
||||
|
||||
it("attaches agent state to a data-only Branch without moving its tip", async () => {
|
||||
const session = await createSession();
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
{
|
||||
kind: "entry",
|
||||
entry: {
|
||||
id: "target",
|
||||
parentId: null,
|
||||
type: "custom",
|
||||
customType: "target",
|
||||
},
|
||||
},
|
||||
setValue(branchTip("main"), "target"),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const harness = await createHarness(session);
|
||||
|
||||
expect(await harness.lanes(BACKGROUND_CONTEXT)).toEqual([]);
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBe("target");
|
||||
expect((await session.getValue(laneConfig("main"), BACKGROUND_CONTEXT))?.value).toEqual({
|
||||
model: { provider: "faux", modelId: "faux-1" },
|
||||
thinkingLevel: "medium",
|
||||
activeToolNames: ["read", "bash"],
|
||||
});
|
||||
});
|
||||
|
||||
it("keeps AgentLane appends operation-aware while exposing the Branch surface directly", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await createHarness(session);
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
const idleId = await lane.appendCustomEntry("idle", undefined, BACKGROUND_CONTEXT);
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBe(idleId);
|
||||
|
||||
const admission = await lane.accept({ kind: "prompt", prompt: "run" }, BACKGROUND_CONTEXT);
|
||||
if (!admission.ok) throw admission.error;
|
||||
const acceptedTip = await lane.getTipId(BACKGROUND_CONTEXT);
|
||||
const pendingId = await lane.appendCustomEntry("pending", { queued: true }, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBe(acceptedTip);
|
||||
expect((await session.getValue(branchTip("main"), BACKGROUND_CONTEXT))?.value).toBe(acceptedTip);
|
||||
expect((await session.getValue(pendingEntry(pendingId), BACKGROUND_CONTEXT))?.value).toEqual({
|
||||
type: "custom",
|
||||
customType: "pending",
|
||||
payload: { queued: true },
|
||||
});
|
||||
expect((await session.getValue(laneState("main"), BACKGROUND_CONTEXT))?.value.inbox).toEqual([
|
||||
{ entryId: pendingId, kind: "write" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("flushes queued writes before a new idle append in one commit", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await createHarness(session);
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
if (!(lane instanceof Lane)) throw new Error("Expected runtime Lane");
|
||||
const first = session.idGenerator.next();
|
||||
const second = session.idGenerator.next();
|
||||
await lane.command((state) => {
|
||||
const inbox = [
|
||||
{ entryId: first, kind: "write" as const },
|
||||
{ entryId: second, kind: "write" as const },
|
||||
];
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [
|
||||
setValue(pendingEntry(first), { type: "custom", customType: "queued-first" }),
|
||||
setValue(pendingEntry(second), { type: "custom", customType: "queued-second" }),
|
||||
setValue(laneState("main"), { currentOperationId: null, lastOperationId: null, inbox }),
|
||||
],
|
||||
next: { ...state, inbox },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
const appended = await lane.appendCustomEntry("new", undefined, BACKGROUND_CONTEXT);
|
||||
|
||||
expect((await lane.findEntries({ order: "oldestFirst" }, BACKGROUND_CONTEXT)).map((entry) => entry.id)).toEqual([
|
||||
first,
|
||||
second,
|
||||
appended,
|
||||
]);
|
||||
expect(lane.state.inbox).toEqual([]);
|
||||
expect((await session.getValue(laneState("main"), BACKGROUND_CONTEXT))?.value.inbox).toEqual([]);
|
||||
expect(await session.getValue(pendingEntry(first), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(await session.getValue(pendingEntry(second), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("restores complete lanes without requiring main", async () => {
|
||||
const session = await createSession();
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
setValue(branchTip("review"), null),
|
||||
setValue(laneConfig("review"), configured),
|
||||
setValue(laneState("review"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const { harness, open } = await AgentHarness.create(harnessOptions(session), BACKGROUND_CONTEXT);
|
||||
expect(open).toEqual([]);
|
||||
expect((await harness.lanes(BACKGROUND_CONTEXT)).map(({ name }) => name)).toEqual(["review"]);
|
||||
expect(await harness.lane("review", BACKGROUND_CONTEXT)).toBeDefined();
|
||||
expect(await session.branch("main", BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
});
|
||||
|
||||
it("rejects partial durable lane state as a Harness fault", async () => {
|
||||
const session = await createSession();
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[setValue(branchTip("main"), null), setValue(laneConfig("main"), configured)],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
await expect(AgentHarness.create(harnessOptions(session), BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(
|
||||
HarnessFault,
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime Harness global metadata", () => {
|
||||
it("preserves value_update publication and delivery", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await createHarness(session);
|
||||
const seen: string[] = [];
|
||||
harness.events.on("value_update", async (event) => {
|
||||
seen.push(event.value === "session_name" ? `name:${await harness.getName(BACKGROUND_CONTEXT)}` : event.value);
|
||||
});
|
||||
|
||||
await harness.setName("named", BACKGROUND_CONTEXT);
|
||||
await harness.setLabel("entry", "label", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await harness.getName(BACKGROUND_CONTEXT)).toBe("named");
|
||||
expect(await harness.getLabel("entry", BACKGROUND_CONTEXT)).toBe("label");
|
||||
expect(seen).toEqual(["name:named", "entry_label"]);
|
||||
});
|
||||
|
||||
it("publishes previous and current data-bearing global configuration", async () => {
|
||||
const harness = await createHarness();
|
||||
const events: unknown[] = [];
|
||||
harness.events.on("config_update", (event) => {
|
||||
events.push(event);
|
||||
});
|
||||
|
||||
await harness.setStreamOptions({ timeoutMs: 123 }, BACKGROUND_CONTEXT);
|
||||
await harness.setSteeringMode("one-at-a-time", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(events).toEqual([
|
||||
expect.objectContaining({
|
||||
type: "config_update",
|
||||
property: "streamOptions",
|
||||
previous: {},
|
||||
value: { timeoutMs: 123 },
|
||||
}),
|
||||
expect.objectContaining({
|
||||
type: "config_update",
|
||||
property: "steeringMode",
|
||||
previous: "all",
|
||||
value: "one-at-a-time",
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it("closes every lane and rejects later acquisition", async () => {
|
||||
const harness = await createHarness();
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
await harness.close(BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(harness.lane("other", BACKGROUND_CONTEXT)).rejects.toThrow("closed");
|
||||
await expect(lane.getTipId(BACKGROUND_CONTEXT)).rejects.toThrow("closed");
|
||||
});
|
||||
|
||||
it("MemorySessionRepo creation also remains branchless", async () => {
|
||||
const repo = new MemorySessionRepo();
|
||||
const session = await repo.create({ id: "repo-session" }, BACKGROUND_CONTEXT);
|
||||
expect(await session.branch("main", BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
391
third_party/pi/packages/agent/test/harness/runtime/lane.test.ts
vendored
Normal file
391
third_party/pi/packages/agent/test/harness/runtime/lane.test.ts
vendored
Normal file
@@ -0,0 +1,391 @@
|
||||
import { type Api, createModels, fauxProvider, type Model } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { HarnessClosed, type HarnessEvent } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT, type Context } from "../../../src/harness/context.ts";
|
||||
import { HarnessEventBus } from "../../../src/harness/events.ts";
|
||||
import { HookRegistry } from "../../../src/harness/hooks.ts";
|
||||
import { convertToLlm } from "../../../src/harness/messages.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { restoreLane } from "../../../src/harness/runtime/restore.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type { LaneConfiguration, Session, StartingOperation } from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import { ControlledMemoryStorage, deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: "test", modelId: "model" },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
};
|
||||
|
||||
async function createLane(
|
||||
emitBatch: (events: readonly HarnessEvent[], context: Context) => Promise<void> = () => Promise.resolve(),
|
||||
): Promise<{
|
||||
lane: Lane<undefined>;
|
||||
model: Model<Api>;
|
||||
session: Session;
|
||||
storage: ControlledMemoryStorage;
|
||||
}> {
|
||||
const storage = new ControlledMemoryStorage();
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `runtime-lane-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const faux = fauxProvider();
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
return {
|
||||
lane: new Lane<undefined>(
|
||||
"main",
|
||||
session,
|
||||
models,
|
||||
new HookRegistry(() => {}),
|
||||
await restoreLane(session, "main", BACKGROUND_CONTEXT),
|
||||
(cause) => (cause instanceof Error ? cause : new Error(String(cause))),
|
||||
emitBatch,
|
||||
(snapshot) => ({
|
||||
snapshot,
|
||||
start: () => {},
|
||||
resnapshot: () => Promise.resolve(snapshot),
|
||||
unsubscribe: () => {},
|
||||
}),
|
||||
() => ({
|
||||
tools: [],
|
||||
resources: {},
|
||||
streamOptions: {},
|
||||
retryPolicy: { enabled: true, maxRetries: 3, baseDelayMs: 1_000 },
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
toolContext: undefined,
|
||||
systemPrompt: undefined,
|
||||
toProviderMessages: (messages) => convertToLlm(messages),
|
||||
entryProjectors: {},
|
||||
}),
|
||||
),
|
||||
model: faux.getModel(),
|
||||
session,
|
||||
storage,
|
||||
};
|
||||
}
|
||||
|
||||
function setThinkingLevel(
|
||||
lane: Lane<undefined>,
|
||||
thinkingLevel: LaneConfiguration["thinkingLevel"],
|
||||
observed?: LaneConfiguration["thinkingLevel"][],
|
||||
): Promise<LaneConfiguration["thinkingLevel"]> {
|
||||
return lane.command((state) => {
|
||||
observed?.push(state.configuration.thinkingLevel);
|
||||
const next = { ...state, configuration: { ...state.configuration, thinkingLevel } };
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.laneConfig(lane.name), next.configuration)],
|
||||
next,
|
||||
materialize: () => thinkingLevel,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime Lane commands", () => {
|
||||
it("reads and replaces configuration from owned state", async () => {
|
||||
const { lane, model, session } = await createLane();
|
||||
const activeToolNames = ["read"];
|
||||
|
||||
await lane.setModel({ provider: model.provider, modelId: model.id }, BACKGROUND_CONTEXT);
|
||||
await lane.setThinkingLevel("high", BACKGROUND_CONTEXT);
|
||||
await lane.setActiveTools(activeToolNames, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(await lane.getModel(BACKGROUND_CONTEXT)).toBe(model);
|
||||
expect(await lane.getThinkingLevel(BACKGROUND_CONTEXT)).toBe("high");
|
||||
expect(await lane.getActiveTools(BACKGROUND_CONTEXT)).toBe(activeToolNames);
|
||||
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value).toEqual({
|
||||
model: { provider: model.provider, modelId: model.id },
|
||||
thinkingLevel: "high",
|
||||
activeToolNames,
|
||||
});
|
||||
});
|
||||
|
||||
it("derives queued configuration updates from the latest committed state", async () => {
|
||||
const { lane, model, storage } = await createLane();
|
||||
const commitStarted = deferred();
|
||||
const releaseCommit = deferred();
|
||||
storage.beforeNextCommit = async () => {
|
||||
commitStarted.resolve();
|
||||
await releaseCommit.promise;
|
||||
};
|
||||
|
||||
const modelUpdate = lane.setModel({ provider: model.provider, modelId: model.id }, BACKGROUND_CONTEXT);
|
||||
await commitStarted.promise;
|
||||
const thinkingUpdate = lane.setThinkingLevel("high", BACKGROUND_CONTEXT);
|
||||
releaseCommit.resolve();
|
||||
await Promise.all([modelUpdate, thinkingUpdate]);
|
||||
|
||||
expect(lane.state.configuration).toEqual({
|
||||
model: { provider: model.provider, modelId: model.id },
|
||||
thinkingLevel: "high",
|
||||
activeToolNames: [],
|
||||
});
|
||||
});
|
||||
|
||||
it("returns a promise value without holding the lane line", async () => {
|
||||
const { lane } = await createLane();
|
||||
const completion = deferred();
|
||||
let completed = false;
|
||||
const joined = lane
|
||||
.command(() => ({ kind: "return", result: completion.promise }), BACKGROUND_CONTEXT)
|
||||
.then(() => {
|
||||
completed = true;
|
||||
});
|
||||
|
||||
await lane.setThinkingLevel("high", BACKGROUND_CONTEXT);
|
||||
expect(completed).toBe(false);
|
||||
completion.resolve();
|
||||
await joined;
|
||||
});
|
||||
|
||||
it("returns an expected rejection without faulting the lane", async () => {
|
||||
const { lane } = await createLane();
|
||||
const rejection = new Error("declined");
|
||||
|
||||
await expect(lane.command(() => ({ kind: "reject", error: rejection }), BACKGROUND_CONTEXT)).rejects.toBe(
|
||||
rejection,
|
||||
);
|
||||
expect(await lane.getTipId(BACKGROUND_CONTEXT)).toBeNull();
|
||||
await lane.setThinkingLevel("high", BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("passes bounded reads and commit metadata through the serialized command", async () => {
|
||||
const { lane } = await createLane();
|
||||
let storedConfiguration: LaneConfiguration | undefined;
|
||||
let memoryPublished = false;
|
||||
|
||||
const commit = await lane.command(async (state, reader) => {
|
||||
storedConfiguration = (await reader.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value;
|
||||
const configuration: LaneConfiguration = { ...state.configuration, thinkingLevel: "high" };
|
||||
const next = { ...state, configuration };
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.laneConfig("main"), configuration)],
|
||||
next,
|
||||
materialize: (commit) => {
|
||||
memoryPublished = lane.state === next;
|
||||
return commit;
|
||||
},
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(storedConfiguration).toEqual(configuration);
|
||||
expect(memoryPublished).toBe(true);
|
||||
expect(commit.seqs).toHaveLength(1);
|
||||
expect(commit.timestamp).toEqual(expect.any(Number));
|
||||
});
|
||||
|
||||
it("rejects thenable materialization after publishing committed memory", async () => {
|
||||
const { lane, session } = await createLane();
|
||||
|
||||
await expect(
|
||||
// @ts-expect-error Exercise the runtime guard against untyped callers.
|
||||
lane.command((state) => {
|
||||
const configuration: LaneConfiguration = { ...state.configuration, thinkingLevel: "high" };
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.laneConfig("main"), configuration)],
|
||||
next: { ...state, configuration },
|
||||
materialize: async () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT),
|
||||
).rejects.toThrow("Lane command materialize() must be synchronous");
|
||||
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("high");
|
||||
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value.thinkingLevel).toBe(
|
||||
"high",
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves committed memory when synchronous event publication fails", async () => {
|
||||
const bus = new HarnessEventBus();
|
||||
const { lane, session } = await createLane((events, context) => bus.emitBatch(events, context));
|
||||
const uncloneable = {
|
||||
type: "run_start",
|
||||
runId: "run",
|
||||
lane: "main",
|
||||
invalid: () => undefined,
|
||||
} as unknown as HarnessEvent;
|
||||
|
||||
await expect(
|
||||
lane.command((state) => {
|
||||
const configuration: LaneConfiguration = { ...state.configuration, thinkingLevel: "high" };
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.laneConfig("main"), configuration)],
|
||||
next: { ...state, configuration },
|
||||
materialize: () => undefined,
|
||||
events: () => [uncloneable],
|
||||
};
|
||||
}, BACKGROUND_CONTEXT),
|
||||
).rejects.toMatchObject({ name: "DataCloneError" });
|
||||
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("high");
|
||||
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value.thinkingLevel).toBe(
|
||||
"high",
|
||||
);
|
||||
});
|
||||
|
||||
it("rejects work after sealing while an admitted commit finishes", async () => {
|
||||
const { lane, storage } = await createLane();
|
||||
const commitStarted = deferred();
|
||||
const releaseCommit = deferred();
|
||||
storage.beforeNextCommit = async () => {
|
||||
commitStarted.resolve();
|
||||
await releaseCommit.promise;
|
||||
};
|
||||
|
||||
const admitted = setThinkingLevel(lane, "high");
|
||||
await commitStarted.promise;
|
||||
const closed = new HarnessClosed();
|
||||
lane.seal(closed);
|
||||
|
||||
await expect(lane.getTipId(BACKGROUND_CONTEXT)).rejects.toBe(closed);
|
||||
await expect(lane.setThinkingLevel("low", BACKGROUND_CONTEXT)).rejects.toBe(closed);
|
||||
releaseCommit.resolve();
|
||||
await admitted;
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("high");
|
||||
});
|
||||
|
||||
it("publishes memory only after the durable commit succeeds", async () => {
|
||||
const { lane, session, storage } = await createLane();
|
||||
const commitStarted = deferred();
|
||||
const releaseCommit = deferred();
|
||||
storage.beforeNextCommit = async () => {
|
||||
commitStarted.resolve();
|
||||
await releaseCommit.promise;
|
||||
};
|
||||
|
||||
const command = setThinkingLevel(lane, "high");
|
||||
await commitStarted.promise;
|
||||
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("off");
|
||||
releaseCommit.resolve();
|
||||
expect(await command).toBe("high");
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("high");
|
||||
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value.thinkingLevel).toBe(
|
||||
"high",
|
||||
);
|
||||
});
|
||||
|
||||
it("preserves memory when the durable commit fails", async () => {
|
||||
const { lane, session, storage } = await createLane();
|
||||
const failure = new Error("commit failed");
|
||||
storage.beforeNextCommit = async () => {
|
||||
throw failure;
|
||||
};
|
||||
|
||||
await expect(setThinkingLevel(lane, "high")).rejects.toBe(failure);
|
||||
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("off");
|
||||
expect((await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT))?.value.thinkingLevel).toBe(
|
||||
"off",
|
||||
);
|
||||
});
|
||||
|
||||
it("diverts ordinary work but settles against latest cancelled control", async () => {
|
||||
const { lane, session } = await createLane();
|
||||
const accepted = await lane.accept({ kind: "prompt", prompt: "hello" }, BACKGROUND_CONTEXT);
|
||||
expect(accepted.ok).toBe(true);
|
||||
const operation = lane.state.operation;
|
||||
if (operation?.state.at !== "starting") throw new Error("missing accepted operation");
|
||||
const cancelled: StartingOperation = {
|
||||
...operation.state,
|
||||
control: { status: "cancel_requested", requestedAt: 1 },
|
||||
};
|
||||
await lane.command(
|
||||
(state) => ({
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(operation.meta.operationId), cancelled)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: cancelled } },
|
||||
materialize: () => undefined,
|
||||
}),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
let continued = false;
|
||||
expect(
|
||||
await lane.continueOperation(
|
||||
cancelled,
|
||||
() => {
|
||||
continued = true;
|
||||
return { kind: "return", result: "continued" };
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual({ kind: "cancel_requested" });
|
||||
expect(continued).toBe(false);
|
||||
|
||||
const settled = await lane.settleOperation(
|
||||
cancelled,
|
||||
(state, current) => {
|
||||
expect(current.control.status).toBe("cancel_requested");
|
||||
const inbox = [...state.inbox, { entryId: "accepted-during-cancellation", kind: "write" as const }];
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [],
|
||||
operationState: current,
|
||||
lane: { inbox },
|
||||
materialize: () => inbox,
|
||||
};
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
expect(settled).toEqual([{ entryId: "accepted-during-cancellation", kind: "write" }]);
|
||||
expect(lane.state.inbox).toEqual(settled);
|
||||
expect((await session.getValue(storedValues.laneState("main"), BACKGROUND_CONTEXT))?.value.inbox).toEqual(
|
||||
settled,
|
||||
);
|
||||
});
|
||||
|
||||
it("plans queued commands from the latest committed memory", async () => {
|
||||
const { lane, storage } = await createLane();
|
||||
const commitStarted = deferred();
|
||||
const releaseCommit = deferred();
|
||||
const observed: LaneConfiguration["thinkingLevel"][] = [];
|
||||
storage.beforeNextCommit = async () => {
|
||||
commitStarted.resolve();
|
||||
await releaseCommit.promise;
|
||||
};
|
||||
|
||||
const first = setThinkingLevel(lane, "high", observed);
|
||||
await commitStarted.promise;
|
||||
const second = setThinkingLevel(lane, "medium", observed);
|
||||
await Promise.resolve();
|
||||
expect(observed).toEqual(["off"]);
|
||||
|
||||
releaseCommit.resolve();
|
||||
expect(await Promise.all([first, second])).toEqual(["high", "medium"]);
|
||||
expect(observed).toEqual(["off", "high"]);
|
||||
expect(lane.state.configuration.thinkingLevel).toBe("medium");
|
||||
});
|
||||
});
|
||||
341
third_party/pi/packages/agent/test/harness/runtime/progress.test.ts
vendored
Normal file
341
third_party/pi/packages/agent/test/harness/runtime/progress.test.ts
vendored
Normal file
@@ -0,0 +1,341 @@
|
||||
import { type AssistantMessageFrame, createModels } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { WatchHandle } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT, type Context, withAbortSignal } from "../../../src/harness/context.ts";
|
||||
import { HookRegistry } from "../../../src/harness/hooks.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import { openFrameProgress, openToolProgress } from "../../../src/harness/runtime/progress.ts";
|
||||
import { type Config, Drive, type LaneState as RuntimeLaneState } from "../../../src/harness/runtime/types.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import {
|
||||
type AssistantEffectPendingOperation,
|
||||
type CheckpointOperation,
|
||||
type OperationState,
|
||||
operationScopeOf,
|
||||
type Session,
|
||||
type Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import { deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const operationId = "operation";
|
||||
const configuration = {
|
||||
model: { provider: "test", modelId: "model" },
|
||||
thinkingLevel: "off" as const,
|
||||
activeToolNames: [],
|
||||
};
|
||||
|
||||
class FailingStorage extends MemoryStorage {
|
||||
failure: Error | undefined;
|
||||
beforeNextCommit: (() => Promise<void>) | undefined;
|
||||
|
||||
override async commit(writes: Write[], context: Context) {
|
||||
const beforeCommit = this.beforeNextCommit;
|
||||
this.beforeNextCommit = undefined;
|
||||
await beforeCommit?.();
|
||||
if (this.failure === undefined) return super.commit(writes, context);
|
||||
const failure = this.failure;
|
||||
this.failure = undefined;
|
||||
throw failure;
|
||||
}
|
||||
}
|
||||
|
||||
const runtimeConfig: Config<undefined> = {
|
||||
tools: [],
|
||||
resources: {},
|
||||
streamOptions: {},
|
||||
retryPolicy: { enabled: true, maxRetries: 3, baseDelayMs: 1_000 },
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
toolContext: undefined,
|
||||
systemPrompt: undefined,
|
||||
toProviderMessages: () => [],
|
||||
entryProjectors: {},
|
||||
};
|
||||
|
||||
function unusedWatch<T>(): WatchHandle<T> {
|
||||
throw new Error("watch is not used by progress tests");
|
||||
}
|
||||
|
||||
function runScope() {
|
||||
return {
|
||||
control: { status: "running" as const },
|
||||
settings: {
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all" as const,
|
||||
followUpMode: "all" as const,
|
||||
toolExecution: "parallel" as const,
|
||||
},
|
||||
inbox: { steer: [], followUp: [], writes: [] },
|
||||
latestAssistantEntryId: null,
|
||||
};
|
||||
}
|
||||
|
||||
function assistantEffectPending(responseEntryId: string): AssistantEffectPendingOperation {
|
||||
return {
|
||||
...runScope(),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: {
|
||||
stepId: "step",
|
||||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 1 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
responseEntryId,
|
||||
usageId: "usage",
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1_000,
|
||||
};
|
||||
}
|
||||
|
||||
async function createFixture(
|
||||
state: OperationState,
|
||||
storage = new FailingStorage(),
|
||||
): Promise<{ lane: Lane<undefined>; drive: Drive; storage: FailingStorage }> {
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `progress-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
const meta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run" as const, promptEntryIds: [] },
|
||||
};
|
||||
const projection: RuntimeLaneState = {
|
||||
tipId: null,
|
||||
configuration,
|
||||
inbox: [],
|
||||
lastOperationId: null,
|
||||
operation: { meta, state },
|
||||
};
|
||||
const lane = new Lane<undefined>(
|
||||
"main",
|
||||
session,
|
||||
createModels(),
|
||||
new HookRegistry(() => {}),
|
||||
projection,
|
||||
(cause) => (cause instanceof Error ? cause : new Error(String(cause))),
|
||||
() => Promise.resolve(),
|
||||
unusedWatch,
|
||||
() => runtimeConfig,
|
||||
);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), state),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const drive = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
lane.activeDrive = drive;
|
||||
return { lane, drive, storage };
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime active drive ownership", () => {
|
||||
it("settles or rejects the shared completion without exposing owner controls through the gate", async () => {
|
||||
const settled = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
const outcome = { kind: "waiting", operationId, reason: "retry", notBefore: 10 } as const;
|
||||
settled.settle(outcome);
|
||||
settled.fail(new Error("late failure"));
|
||||
await expect(settled.completion).resolves.toBe(outcome);
|
||||
|
||||
const failed = new Drive({ operationId }, BACKGROUND_CONTEXT);
|
||||
const error = new Error("drive failed");
|
||||
failed.fail(error);
|
||||
await expect(failed.completion).rejects.toBe(error);
|
||||
expect("beginAbort" in failed.gate).toBe(false);
|
||||
});
|
||||
|
||||
it("strips invocation cancellation from pass context and owns policy and gate control", () => {
|
||||
const controller = new AbortController();
|
||||
const drive = new Drive(
|
||||
{ operationId, waitForRetry: true, pollDeferred: true },
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
);
|
||||
|
||||
expect(drive.operationId).toBe(operationId);
|
||||
expect(drive.context.abortSignal).toBeUndefined();
|
||||
expect(drive.waitForRetry).toBe(true);
|
||||
expect(drive.deferredPermits).toBe(1);
|
||||
|
||||
const closed = new Error("closed");
|
||||
drive.closeGate(closed);
|
||||
expect(drive.gate.signal.aborted).toBe(true);
|
||||
expect(() => drive.gate.admit(() => undefined)).toThrow(closed);
|
||||
});
|
||||
});
|
||||
|
||||
describe("runtime progress channels", () => {
|
||||
it("enqueues assistant frames in order, seals admission, and exposes the settlement delete", async () => {
|
||||
const responseEntryId = "response";
|
||||
const { lane, drive } = await createFixture(assistantEffectPending(responseEntryId));
|
||||
const progress = openFrameProgress(lane, drive, responseEntryId);
|
||||
const frames: AssistantMessageFrame[] = [
|
||||
{ type: "text_delta", contentIndex: 0, delta: "a" },
|
||||
{ type: "text_delta", contentIndex: 0, delta: "b" },
|
||||
];
|
||||
|
||||
progress.write(frames[0]!);
|
||||
progress.write(frames[1]!);
|
||||
progress.seal();
|
||||
progress.write({ type: "text_delta", contentIndex: 0, delta: "late" });
|
||||
await progress.drain();
|
||||
|
||||
expect(
|
||||
(
|
||||
await lane.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
)
|
||||
).map(({ value }) => value),
|
||||
).toEqual(frames);
|
||||
});
|
||||
|
||||
it("declines a queued frame after the authoritative projection leaves its phase without control reads", async () => {
|
||||
const responseEntryId = "response";
|
||||
const { lane, drive, storage } = await createFixture(assistantEffectPending(responseEntryId));
|
||||
const progress = openFrameProgress(lane, drive, responseEntryId);
|
||||
const commitStarted = deferred();
|
||||
const releaseCommit = deferred();
|
||||
storage.beforeNextCommit = async () => {
|
||||
commitStarted.resolve();
|
||||
await releaseCommit.promise;
|
||||
};
|
||||
const getValue = vi.spyOn(storage, "getValue");
|
||||
const moving = lane.command((state) => {
|
||||
const operation = state.operation;
|
||||
if (operation === null) throw new Error("missing operation");
|
||||
const nextState: CheckpointOperation = {
|
||||
...operationScopeOf(operation.state),
|
||||
at: "checkpoint",
|
||||
continuation: { kind: "may_finish", includeFinalAssistant: true },
|
||||
triggerEntryId: responseEntryId,
|
||||
};
|
||||
return {
|
||||
kind: "commit",
|
||||
writes: [storedValues.setValue(storedValues.operationState(operationId), nextState)],
|
||||
next: { ...state, operation: { meta: operation.meta, state: nextState } },
|
||||
materialize: () => undefined,
|
||||
};
|
||||
}, BACKGROUND_CONTEXT);
|
||||
await commitStarted.promise;
|
||||
progress.write({ type: "text_delta", contentIndex: 0, delta: "late" });
|
||||
releaseCommit.resolve();
|
||||
await moving;
|
||||
await progress.drain();
|
||||
|
||||
expect(
|
||||
await lane.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(getValue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("declines a queued frame after terminal projection publication", async () => {
|
||||
const responseEntryId = "response";
|
||||
const { lane, drive, storage } = await createFixture(assistantEffectPending(responseEntryId));
|
||||
const progress = openFrameProgress(lane, drive, responseEntryId);
|
||||
const commitStarted = deferred();
|
||||
const releaseCommit = deferred();
|
||||
storage.beforeNextCommit = async () => {
|
||||
commitStarted.resolve();
|
||||
await releaseCommit.promise;
|
||||
};
|
||||
const getValue = vi.spyOn(storage, "getValue");
|
||||
const ending = lane.command(
|
||||
(state) => ({
|
||||
kind: "commit",
|
||||
writes: [
|
||||
storedValues.deleteValue(storedValues.operationMeta(operationId)),
|
||||
storedValues.deleteValue(storedValues.operationState(operationId)),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: state.inbox,
|
||||
}),
|
||||
],
|
||||
next: { ...state, operation: null },
|
||||
materialize: () => undefined,
|
||||
}),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await commitStarted.promise;
|
||||
progress.write({ type: "text_delta", contentIndex: 0, delta: "late" });
|
||||
releaseCommit.resolve();
|
||||
await ending;
|
||||
await progress.drain();
|
||||
|
||||
expect(
|
||||
await lane.session.readList(
|
||||
storedValues.pendingAssistantFrames(operationId, responseEntryId),
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).toEqual([]);
|
||||
expect(getValue).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("replaces tool checkpoints in invocation order", async () => {
|
||||
const invocationId = "result";
|
||||
const { lane, drive } = await createFixture({
|
||||
...runScope(),
|
||||
at: "tools",
|
||||
batch: {
|
||||
assistantEntryId: "assistant",
|
||||
configuration,
|
||||
turnId: "turn",
|
||||
calls: [{ status: "effect_pending", sourceIndex: 0, resultEntryId: invocationId, replay: "safe" }],
|
||||
},
|
||||
} satisfies OperationState);
|
||||
const progress = openToolProgress(lane, drive, "turn", 0, invocationId);
|
||||
progress.write({ content: [{ type: "text", text: "first" }], details: {} });
|
||||
progress.write({ content: [{ type: "text", text: "second" }], details: {} });
|
||||
await progress.drain();
|
||||
expect(
|
||||
(await lane.session.getValue(storedValues.pendingToolOutput(operationId, invocationId), BACKGROUND_CONTEXT))
|
||||
?.value.content,
|
||||
).toEqual([{ type: "text", text: "second" }]);
|
||||
});
|
||||
|
||||
it("retains the rejecting write promise so drain propagates commit failure", async () => {
|
||||
const responseEntryId = "response";
|
||||
const { lane, drive, storage } = await createFixture(assistantEffectPending(responseEntryId));
|
||||
const failure = new Error("frame commit failed");
|
||||
storage.failure = failure;
|
||||
const progress = openFrameProgress(lane, drive, responseEntryId);
|
||||
|
||||
progress.write({ type: "text_delta", contentIndex: 0, delta: "lost" });
|
||||
|
||||
await expect(progress.drain()).rejects.toBe(failure);
|
||||
});
|
||||
});
|
||||
295
third_party/pi/packages/agent/test/harness/runtime/reducer.test.ts
vendored
Normal file
295
third_party/pi/packages/agent/test/harness/runtime/reducer.test.ts
vendored
Normal file
@@ -0,0 +1,295 @@
|
||||
import { createModels, fauxAssistantMessage, fauxProvider } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { AgentHarness, type HarnessEvent, type LaneSnapshot } from "../../../src/harness/agent-harness.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { reduceLaneSnapshot } from "../../../src/harness/runtime/reducer.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type { Session } from "../../../src/harness/session/types.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
|
||||
async function createFixture(options: { deferred?: boolean } = {}) {
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `reducer-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
new MemoryStorage(),
|
||||
);
|
||||
sessions.push(session);
|
||||
const faux = fauxProvider();
|
||||
const models = createModels();
|
||||
models.setProvider(faux.provider);
|
||||
const { harness } = await AgentHarness.create(
|
||||
{
|
||||
session,
|
||||
models,
|
||||
model: faux.getModel(),
|
||||
...(options.deferred === true ? { streamOptions: { deferred: true } } : {}),
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
return { harness, lane, faux };
|
||||
}
|
||||
|
||||
async function settleEvents(): Promise<void> {
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
function fold(snapshot: LaneSnapshot, events: HarnessEvent[]): LaneSnapshot {
|
||||
for (const event of events) {
|
||||
if (reduceLaneSnapshot(snapshot, event) === "rebase") throw new Error(`Unexpected rebase for ${event.type}`);
|
||||
}
|
||||
return snapshot;
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("lane snapshot reducer", () => {
|
||||
it("folds an ordinary run to the authoritative resnapshot", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
const watch = await lane.watch(BACKGROUND_CONTEXT);
|
||||
const events: HarnessEvent[] = [];
|
||||
watch.start((event) => {
|
||||
events.push(event);
|
||||
});
|
||||
faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
|
||||
expect(await lane.prompt("question", undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "run", status: "completed" },
|
||||
});
|
||||
await settleEvents();
|
||||
|
||||
expect(fold(watch.snapshot, events)).toEqual(await watch.resnapshot(BACKGROUND_CONTEXT));
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("folds suspend and resume without closing the operation early", async () => {
|
||||
const { lane, faux } = await createFixture({ deferred: true });
|
||||
const watch = await lane.watch(BACKGROUND_CONTEXT);
|
||||
const events: HarnessEvent[] = [];
|
||||
watch.start((event) => {
|
||||
events.push(event);
|
||||
});
|
||||
faux.setResponses([fauxAssistantMessage("answer")]);
|
||||
|
||||
expect(await lane.prompt("question", undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { status: "suspended" },
|
||||
});
|
||||
await settleEvents();
|
||||
let replica = fold(watch.snapshot, events);
|
||||
expect(replica.operation).toMatchObject({ kind: "run", deferred: { poll: 0 } });
|
||||
expect(replica).toEqual(await watch.resnapshot(BACKGROUND_CONTEXT));
|
||||
|
||||
events.length = 0;
|
||||
expect(await lane.resume(BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { kind: "run", status: "completed" },
|
||||
});
|
||||
await settleEvents();
|
||||
replica = fold(watch.snapshot, events);
|
||||
expect(replica).toEqual(await watch.resnapshot(BACKGROUND_CONTEXT));
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("folds standalone compaction and preserves segment semantics", async () => {
|
||||
const { lane, faux } = await createFixture();
|
||||
await lane.appendMessage({ role: "user", content: "history", timestamp: 1 }, BACKGROUND_CONTEXT);
|
||||
const watch = await lane.watch(BACKGROUND_CONTEXT);
|
||||
const events: HarnessEvent[] = [];
|
||||
watch.start((event) => {
|
||||
events.push(event);
|
||||
});
|
||||
faux.setResponses([fauxAssistantMessage("summary")]);
|
||||
|
||||
expect(await lane.compact(undefined, BACKGROUND_CONTEXT)).toMatchObject({
|
||||
ok: true,
|
||||
value: { compaction: { status: "completed" } },
|
||||
});
|
||||
await settleEvents();
|
||||
const replica = fold(watch.snapshot, events);
|
||||
|
||||
expect(replica.operation).toBeNull();
|
||||
expect(replica.lastResult).toMatchObject({ kind: "compaction", status: "completed" });
|
||||
expect(replica).toEqual(await watch.resnapshot(BACKGROUND_CONTEXT));
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("replicates globally ordered queue changes", async () => {
|
||||
const { lane } = await createFixture();
|
||||
const watch = await lane.watch(BACKGROUND_CONTEXT);
|
||||
const events: HarnessEvent[] = [];
|
||||
watch.start((event) => {
|
||||
events.push(event);
|
||||
});
|
||||
|
||||
await lane.nextRun("next", undefined, BACKGROUND_CONTEXT);
|
||||
const steer = await lane.steer("steer", undefined, BACKGROUND_CONTEXT);
|
||||
await lane.followUp("follow", undefined, BACKGROUND_CONTEXT);
|
||||
if (!steer.ok) throw steer.error;
|
||||
await lane.cancelQueued(steer.value.entryId, BACKGROUND_CONTEXT);
|
||||
await settleEvents();
|
||||
|
||||
const replica = fold(watch.snapshot, events);
|
||||
expect(replica.queues.map((item) => item.kind)).toEqual(["nextRun", "followUp"]);
|
||||
expect(replica).toEqual(await watch.resnapshot(BACKGROUND_CONTEXT));
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("keeps in-run compaction segments inside the open run", async () => {
|
||||
const { lane } = await createFixture();
|
||||
const snapshot = (await lane.watch(BACKGROUND_CONTEXT)).snapshot;
|
||||
const running: LaneSnapshot = {
|
||||
...snapshot,
|
||||
operation: {
|
||||
id: "run",
|
||||
kind: "run",
|
||||
startedAt: 1,
|
||||
fromTipId: null,
|
||||
status: "open",
|
||||
runningTools: [],
|
||||
},
|
||||
};
|
||||
expect(
|
||||
reduceLaneSnapshot(running, {
|
||||
type: "compaction_start",
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
reason: "threshold",
|
||||
startedAt: 2,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(
|
||||
reduceLaneSnapshot(running, {
|
||||
type: "compaction_end",
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
reason: "threshold",
|
||||
status: "declined",
|
||||
endedAt: 3,
|
||||
}),
|
||||
).toBeUndefined();
|
||||
expect(running).toMatchObject({ operation: { id: "run", kind: "run" } });
|
||||
});
|
||||
|
||||
it("retains settled parallel tools until each source-ordered result is placed", async () => {
|
||||
const { lane } = await createFixture();
|
||||
const snapshot = (await lane.watch(BACKGROUND_CONTEXT)).snapshot;
|
||||
snapshot.operation = {
|
||||
id: "run",
|
||||
kind: "run",
|
||||
startedAt: 1,
|
||||
fromTipId: null,
|
||||
status: "open",
|
||||
runningTools: [],
|
||||
};
|
||||
const result = (text: string) => ({ content: [{ type: "text" as const, text }], details: { text } });
|
||||
const start = (index: number): HarnessEvent => ({
|
||||
type: "tool_start",
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
turnId: "turn",
|
||||
toolCallId: `call-${index}`,
|
||||
toolName: `tool-${index}`,
|
||||
args: { index },
|
||||
});
|
||||
const end = (index: number): HarnessEvent => ({
|
||||
type: "tool_end",
|
||||
lane: "main",
|
||||
runId: "run",
|
||||
turnId: "turn",
|
||||
toolCallId: `call-${index}`,
|
||||
toolName: `tool-${index}`,
|
||||
result: result(`done-${index}`),
|
||||
isError: false,
|
||||
terminate: false,
|
||||
});
|
||||
const entry = (index: number): HarnessEvent => ({
|
||||
type: "entry_added",
|
||||
lane: "main",
|
||||
entry: {
|
||||
id: `result-${index}`,
|
||||
parentId: index === 0 ? null : `result-${index - 1}`,
|
||||
seq: index + 1,
|
||||
timestamp: index + 1,
|
||||
type: "message",
|
||||
message: {
|
||||
role: "toolResult",
|
||||
toolCallId: `call-${index}`,
|
||||
toolName: `tool-${index}`,
|
||||
content: [{ type: "text", text: `done-${index}` }],
|
||||
isError: false,
|
||||
timestamp: index + 1,
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
for (let index = 0; index < 3; index += 1) reduceLaneSnapshot(snapshot, start(index));
|
||||
expect(snapshot.operation.runningTools).toHaveLength(3);
|
||||
reduceLaneSnapshot(snapshot, start(0));
|
||||
expect(snapshot.operation.runningTools).toHaveLength(3);
|
||||
|
||||
reduceLaneSnapshot(snapshot, {
|
||||
type: "tool_update",
|
||||
lane: "main",
|
||||
runId: "stale",
|
||||
turnId: "turn",
|
||||
toolCallId: "call-1",
|
||||
toolName: "tool-1",
|
||||
partialResult: result("stale"),
|
||||
});
|
||||
expect(snapshot.operation.runningTools[1]).not.toHaveProperty("result");
|
||||
|
||||
reduceLaneSnapshot(snapshot, end(2));
|
||||
expect(snapshot.operation.runningTools.find(({ toolCallId }) => toolCallId === "call-2")).toMatchObject({
|
||||
status: "settled",
|
||||
args: { index: 2 },
|
||||
result: result("done-2"),
|
||||
});
|
||||
|
||||
reduceLaneSnapshot(snapshot, end(0));
|
||||
expect(snapshot.operation.runningTools.map(({ status }) => status)).toEqual(["settled", "running", "settled"]);
|
||||
reduceLaneSnapshot(snapshot, entry(0));
|
||||
expect(snapshot.operation.runningTools.map(({ toolCallId }) => toolCallId)).toEqual(["call-1", "call-2"]);
|
||||
expect(snapshot.transcript.map(({ id }) => id)).toEqual(["result-0"]);
|
||||
|
||||
reduceLaneSnapshot(snapshot, end(1));
|
||||
expect(snapshot.operation.runningTools.every(({ status }) => status === "settled")).toBe(true);
|
||||
reduceLaneSnapshot(snapshot, entry(1));
|
||||
expect(snapshot.operation.runningTools.map(({ toolCallId }) => toolCallId)).toEqual(["call-2"]);
|
||||
reduceLaneSnapshot(snapshot, entry(2));
|
||||
expect(snapshot.operation.runningTools).toEqual([]);
|
||||
expect(snapshot.transcript.map(({ id }) => id)).toEqual(["result-0", "result-1", "result-2"]);
|
||||
});
|
||||
|
||||
it("marks navigation completion for rebase", async () => {
|
||||
const { lane } = await createFixture();
|
||||
const snapshot = (await lane.watch(BACKGROUND_CONTEXT)).snapshot;
|
||||
const reduced = reduceLaneSnapshot(
|
||||
{
|
||||
...snapshot,
|
||||
operation: {
|
||||
id: "navigation",
|
||||
kind: "navigation",
|
||||
startedAt: 1,
|
||||
fromTipId: null,
|
||||
status: "open",
|
||||
runningTools: [],
|
||||
},
|
||||
},
|
||||
{
|
||||
type: "navigation_end",
|
||||
lane: "main",
|
||||
runId: "navigation",
|
||||
status: "completed",
|
||||
fromTipId: null,
|
||||
tipId: "target",
|
||||
endedAt: 2,
|
||||
},
|
||||
);
|
||||
expect(reduced).toBe("rebase");
|
||||
});
|
||||
});
|
||||
441
third_party/pi/packages/agent/test/harness/runtime/restore.test.ts
vendored
Normal file
441
third_party/pi/packages/agent/test/harness/runtime/restore.test.ts
vendored
Normal file
@@ -0,0 +1,441 @@
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { restoreLane, restoreSession } from "../../../src/harness/runtime/restore.ts";
|
||||
import { MemorySessionRepo, MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import type {
|
||||
CheckpointOperation,
|
||||
LaneConfiguration,
|
||||
OperationMeta,
|
||||
OperationScope,
|
||||
OperationState,
|
||||
ResultBoundary,
|
||||
Session,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
|
||||
const repos: MemorySessionRepo[] = [];
|
||||
const configuration: LaneConfiguration = {
|
||||
model: { provider: "test", modelId: "model" },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: [],
|
||||
};
|
||||
|
||||
async function createSession(): Promise<Session> {
|
||||
const repo = new MemorySessionRepo();
|
||||
repos.push(repo);
|
||||
const session = await repo.create({}, BACKGROUND_CONTEXT);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return session;
|
||||
}
|
||||
|
||||
function operationScope(): OperationScope {
|
||||
return {
|
||||
control: { status: "running" },
|
||||
settings: {
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
},
|
||||
latestAssistantEntryId: null,
|
||||
};
|
||||
}
|
||||
|
||||
function runState(triggerEntryId: string): CheckpointOperation {
|
||||
return {
|
||||
...operationScope(),
|
||||
at: "checkpoint",
|
||||
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
|
||||
triggerEntryId,
|
||||
};
|
||||
}
|
||||
|
||||
function summaryState(boundary: ResultBoundary): OperationState {
|
||||
return {
|
||||
...operationScope(),
|
||||
at: "summary.deciding",
|
||||
task: {
|
||||
taskId: "task",
|
||||
...(boundary.kind === "finish"
|
||||
? { reason: "manual" as const }
|
||||
: boundary.kind === "resume_checkpoint"
|
||||
? { reason: "threshold" as const }
|
||||
: {}),
|
||||
boundary,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const repo of repos.splice(0)) await repo.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime lane restore", () => {
|
||||
it("restores an idle lane's latest operation id without reading its result", async () => {
|
||||
const session = await createSession();
|
||||
const result = {
|
||||
operationId: session.idGenerator.next(),
|
||||
kind: "navigation" as const,
|
||||
status: "completed" as const,
|
||||
fromTipId: null,
|
||||
tipId: null,
|
||||
startedAt: 1,
|
||||
endedAt: 2,
|
||||
};
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.operationResult(result.operationId), result),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: result.operationId,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const state = await restoreLane(session, "main", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(state).toEqual({
|
||||
tipId: null,
|
||||
configuration,
|
||||
inbox: [],
|
||||
lastOperationId: result.operationId,
|
||||
operation: null,
|
||||
});
|
||||
});
|
||||
|
||||
it("restores an open operation without interpreting its referenced payloads", async () => {
|
||||
const session = await createSession();
|
||||
const operationId = session.idGenerator.next();
|
||||
const missingTriggerId = session.idGenerator.next();
|
||||
const meta: OperationMeta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
};
|
||||
const state = runState(missingTriggerId);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), state),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const restored = await restoreLane(session, "main", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(restored.operation).toEqual({ meta, state });
|
||||
});
|
||||
|
||||
it("validates current-operation identity, lane ownership, and intent/state compatibility", async () => {
|
||||
for (const corruption of ["identity", "lane", "kind"] as const) {
|
||||
const session = await createSession();
|
||||
const operationId = session.idGenerator.next();
|
||||
const base: OperationMeta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
};
|
||||
const meta: OperationMeta =
|
||||
corruption === "identity"
|
||||
? { ...base, operationId: "different-operation" }
|
||||
: corruption === "lane"
|
||||
? { ...base, lane: "worker" }
|
||||
: { ...base, intent: { kind: "navigation", targetId: null, summarize: false } };
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), runState("trigger")),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
await expect(restoreLane(session, "main", BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(Error);
|
||||
}
|
||||
});
|
||||
|
||||
it("accepts exactly the family-neutral state reachability matrix", async () => {
|
||||
const resume = { kind: "resume_checkpoint", resumeAfter: runState("trigger") } satisfies ResultBoundary;
|
||||
const finish = { kind: "finish" } satisfies ResultBoundary;
|
||||
const navigation = { kind: "commit_navigation", targetId: "target" } satisfies ResultBoundary;
|
||||
const cases: { intent: OperationMeta["intent"]; state: OperationState; accepted: boolean }[] = [
|
||||
{ intent: { kind: "run", promptEntryIds: [] }, state: runState("trigger"), accepted: true },
|
||||
{ intent: { kind: "run", promptEntryIds: [] }, state: summaryState(resume), accepted: true },
|
||||
{ intent: { kind: "run", promptEntryIds: [] }, state: summaryState(finish), accepted: false },
|
||||
{ intent: { kind: "run", promptEntryIds: [] }, state: summaryState(navigation), accepted: false },
|
||||
{ intent: { kind: "compaction" }, state: summaryState(finish), accepted: true },
|
||||
{ intent: { kind: "compaction" }, state: summaryState(resume), accepted: false },
|
||||
{ intent: { kind: "compaction" }, state: summaryState(navigation), accepted: false },
|
||||
{ intent: { kind: "compaction" }, state: runState("trigger"), accepted: false },
|
||||
{
|
||||
intent: { kind: "navigation", targetId: null, summarize: false },
|
||||
state: { ...operationScope(), at: "navigation.ready_to_commit", targetId: null },
|
||||
accepted: true,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: false },
|
||||
state: { ...operationScope(), at: "navigation.ready_to_commit", targetId: "different" },
|
||||
accepted: false,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: false },
|
||||
state: summaryState(navigation),
|
||||
accepted: false,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: true },
|
||||
state: summaryState(navigation),
|
||||
accepted: true,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "different", summarize: true },
|
||||
state: summaryState(navigation),
|
||||
accepted: false,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: true },
|
||||
state: { ...operationScope(), at: "navigation.ready_to_commit", targetId: "target" },
|
||||
accepted: false,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: true },
|
||||
state: summaryState(finish),
|
||||
accepted: false,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: true },
|
||||
state: summaryState(resume),
|
||||
accepted: false,
|
||||
},
|
||||
{
|
||||
intent: { kind: "navigation", targetId: "target", summarize: false },
|
||||
state: runState("trigger"),
|
||||
accepted: false,
|
||||
},
|
||||
];
|
||||
|
||||
for (const testCase of cases) {
|
||||
const session = await createSession();
|
||||
const operationId = session.idGenerator.next();
|
||||
const meta: OperationMeta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: testCase.intent,
|
||||
};
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), testCase.state),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const restored = restoreLane(session, "main", BACKGROUND_CONTEXT);
|
||||
if (testCase.accepted)
|
||||
await expect(restored).resolves.toMatchObject({ operation: { meta, state: testCase.state } });
|
||||
else await expect(restored).rejects.toThrow("does not match state");
|
||||
}
|
||||
});
|
||||
|
||||
it.each([
|
||||
[storedValues.branchTip("main").namespace, storedValues.deleteValue(storedValues.branchTip("main"))],
|
||||
[storedValues.laneConfig("main").namespace, storedValues.deleteValue(storedValues.laneConfig("main"))],
|
||||
[storedValues.laneState("main").namespace, storedValues.deleteValue(storedValues.laneState("main"))],
|
||||
] as const)("requires %s", async (namespace, write) => {
|
||||
const session = await createSession();
|
||||
await session.mutate((mutator) => mutator.commit([write], BACKGROUND_CONTEXT), BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(restoreLane(session, "main", BACKGROUND_CONTEXT)).rejects.toThrow(`missing ${namespace.slice(3)}`);
|
||||
});
|
||||
|
||||
it.each([storedValues.operationMeta("").namespace, storedValues.operationState("").namespace] as const)(
|
||||
"requires %s for the current operation",
|
||||
async (namespace) => {
|
||||
const session = await createSession();
|
||||
const operationId = session.idGenerator.next();
|
||||
const meta: OperationMeta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
};
|
||||
const state = runState(session.idGenerator.next());
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
...(namespace === storedValues.operationMeta(operationId).namespace
|
||||
? []
|
||||
: [storedValues.setValue(storedValues.operationMeta(operationId), meta)]),
|
||||
...(namespace === storedValues.operationState(operationId).namespace
|
||||
? []
|
||||
: [storedValues.setValue(storedValues.operationState(operationId), state)]),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
await expect(restoreLane(session, "main", BACKGROUND_CONTEXT)).rejects.toThrow(
|
||||
`missing ${namespace.slice(3)}`,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
it("restores every configured lane exactly once without writing", async () => {
|
||||
const session = await createSession();
|
||||
const workerConfiguration: LaneConfiguration = {
|
||||
model: { provider: "test", modelId: "worker" },
|
||||
thinkingLevel: "high",
|
||||
activeToolNames: ["read"],
|
||||
};
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("worker"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("worker"), workerConfiguration),
|
||||
storedValues.setValue(storedValues.laneState("worker"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const operationId = session.idGenerator.next();
|
||||
const meta: OperationMeta = {
|
||||
operationId,
|
||||
lane: "worker",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
};
|
||||
const state = runState(session.idGenerator.next());
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), state),
|
||||
storedValues.setValue(storedValues.laneState("worker"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const before = {
|
||||
leaves: await session.scanValues(storedValues.branchTipInventoryPrefix(), BACKGROUND_CONTEXT),
|
||||
mainConfiguration: await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT),
|
||||
workerConfiguration: await session.getValue(storedValues.laneConfig("worker"), BACKGROUND_CONTEXT),
|
||||
mainState: await session.getValue(storedValues.laneState("main"), BACKGROUND_CONTEXT),
|
||||
workerState: await session.getValue(storedValues.laneState("worker"), BACKGROUND_CONTEXT),
|
||||
};
|
||||
const mutate = vi.spyOn(session, "mutate");
|
||||
const getValue = vi.spyOn(MemoryStorage.prototype, "getValue");
|
||||
const readList = vi.spyOn(MemoryStorage.prototype, "readList");
|
||||
|
||||
const lanes = await restoreSession(session, BACKGROUND_CONTEXT);
|
||||
|
||||
expect([...lanes.keys()].sort()).toEqual(["main", "worker"]);
|
||||
expect(lanes.get("main")?.configuration).toEqual(configuration);
|
||||
expect(lanes.get("worker")).toMatchObject({
|
||||
configuration: workerConfiguration,
|
||||
operation: { meta, state },
|
||||
});
|
||||
expect(mutate).toHaveBeenCalledTimes(1);
|
||||
expect(getValue.mock.calls.map(([address]) => ({ namespace: address.namespace, key: address.key }))).toEqual([
|
||||
{ namespace: storedValues.operationMeta("").namespace, key: operationId },
|
||||
{ namespace: storedValues.operationState("").namespace, key: operationId },
|
||||
]);
|
||||
expect(readList).not.toHaveBeenCalled();
|
||||
getValue.mockRestore();
|
||||
expect({
|
||||
leaves: await session.scanValues(storedValues.branchTipInventoryPrefix(), BACKGROUND_CONTEXT),
|
||||
mainConfiguration: await session.getValue(storedValues.laneConfig("main"), BACKGROUND_CONTEXT),
|
||||
workerConfiguration: await session.getValue(storedValues.laneConfig("worker"), BACKGROUND_CONTEXT),
|
||||
mainState: await session.getValue(storedValues.laneState("main"), BACKGROUND_CONTEXT),
|
||||
workerState: await session.getValue(storedValues.laneState("worker"), BACKGROUND_CONTEXT),
|
||||
}).toEqual(before);
|
||||
readList.mockRestore();
|
||||
});
|
||||
|
||||
it("allows an empty inventory but rejects lane values without a Branch", async () => {
|
||||
const repo = new MemorySessionRepo();
|
||||
repos.push(repo);
|
||||
const empty = await repo.create({}, BACKGROUND_CONTEXT);
|
||||
await expect(restoreSession(empty, BACKGROUND_CONTEXT)).resolves.toEqual(new Map());
|
||||
|
||||
const session = await createSession();
|
||||
await session.mutate(
|
||||
(mutator) => mutator.commit([storedValues.deleteValue(storedValues.branchTip("main"))], BACKGROUND_CONTEXT),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await expect(restoreSession(session, BACKGROUND_CONTEXT)).rejects.toThrow('Lane "main" is missing branch.tip');
|
||||
});
|
||||
});
|
||||
32
third_party/pi/packages/agent/test/harness/runtime/test-utils.ts
vendored
Normal file
32
third_party/pi/packages/agent/test/harness/runtime/test-utils.ts
vendored
Normal file
@@ -0,0 +1,32 @@
|
||||
import { BACKGROUND_CONTEXT } from "../../../src/harness/context.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import type { CommitResult, Write } from "../../../src/harness/session/types.ts";
|
||||
|
||||
export class FailingMemoryStorage extends MemoryStorage {
|
||||
failure: Error | undefined;
|
||||
|
||||
override commit(writes: Write[]) {
|
||||
const failure = this.failure;
|
||||
this.failure = undefined;
|
||||
return failure === undefined ? super.commit(writes, BACKGROUND_CONTEXT) : Promise.reject(failure);
|
||||
}
|
||||
}
|
||||
|
||||
export class ControlledMemoryStorage extends MemoryStorage {
|
||||
beforeNextCommit: (() => Promise<void>) | undefined;
|
||||
|
||||
override async commit(writes: Write[]): Promise<CommitResult> {
|
||||
const beforeCommit = this.beforeNextCommit;
|
||||
this.beforeNextCommit = undefined;
|
||||
await beforeCommit?.();
|
||||
return super.commit(writes, BACKGROUND_CONTEXT);
|
||||
}
|
||||
}
|
||||
|
||||
export function deferred(): { promise: Promise<void>; resolve(): void } {
|
||||
let resolvePromise: (() => void) | undefined;
|
||||
const promise = new Promise<void>((resolve) => {
|
||||
resolvePromise = resolve;
|
||||
});
|
||||
return { promise, resolve: () => resolvePromise?.() };
|
||||
}
|
||||
595
third_party/pi/packages/agent/test/harness/runtime/watch.test.ts
vendored
Normal file
595
third_party/pi/packages/agent/test/harness/runtime/watch.test.ts
vendored
Normal file
@@ -0,0 +1,595 @@
|
||||
import { type AssistantMessageFrame, createModels, fauxAssistantMessage, fauxProvider } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type HarnessEvent, HarnessFault, type WatchHandle } from "../../../src/harness/agent-harness.ts";
|
||||
import { DEFAULT_COMPACTION_SETTINGS } from "../../../src/harness/compaction/compaction.ts";
|
||||
import { BACKGROUND_CONTEXT, type Context, createContextKey, withContextValue } from "../../../src/harness/context.ts";
|
||||
import { createAgentHarness, Harness } from "../../../src/harness/runtime/harness.ts";
|
||||
import { Lane } from "../../../src/harness/runtime/lane.ts";
|
||||
import * as sessionWrites from "../../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../../src/harness/session/session.ts";
|
||||
import type {
|
||||
OperationMeta,
|
||||
OperationScope,
|
||||
OperationState,
|
||||
Session,
|
||||
StorageBranchScan,
|
||||
Write,
|
||||
} from "../../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../../src/harness/session/values.ts";
|
||||
import { deferred } from "./test-utils.ts";
|
||||
|
||||
const sessions: Session[] = [];
|
||||
const configuration = {
|
||||
model: { provider: "configured", modelId: "model" },
|
||||
thinkingLevel: "off" as const,
|
||||
activeToolNames: [],
|
||||
};
|
||||
|
||||
async function createSession(storage: MemoryStorage = new MemoryStorage()): Promise<Session> {
|
||||
const session = new StorageBackedSession(
|
||||
{ id: `watch-${sessions.length}`, createdAt: 1, storageVersion: 1 },
|
||||
storage,
|
||||
);
|
||||
sessions.push(session);
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
storedValues.setValue(storedValues.branchTip("main"), null),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return session;
|
||||
}
|
||||
|
||||
async function commit(session: Session, writes: Write[]): Promise<void> {
|
||||
await session.mutate((mutator) => mutator.commit(writes, BACKGROUND_CONTEXT), BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
function runScope(control: OperationScope["control"] = { status: "running" }): OperationScope {
|
||||
return {
|
||||
control,
|
||||
settings: {
|
||||
compaction: DEFAULT_COMPACTION_SETTINGS,
|
||||
steeringMode: "all",
|
||||
followUpMode: "all",
|
||||
toolExecution: "parallel",
|
||||
},
|
||||
latestAssistantEntryId: null,
|
||||
};
|
||||
}
|
||||
|
||||
async function attach(
|
||||
session: Session,
|
||||
): Promise<Lane<object | undefined> & Pick<Harness<object | undefined>, "events">> {
|
||||
const provider = fauxProvider();
|
||||
const models = createModels();
|
||||
models.setProvider(provider.provider);
|
||||
const { harness } = await createAgentHarness({ session, models, model: provider.getModel() }, BACKGROUND_CONTEXT);
|
||||
if (!(harness instanceof Harness)) throw new Error("Expected runtime Harness");
|
||||
const lane = await harness.lane("main", BACKGROUND_CONTEXT);
|
||||
if (!(lane instanceof Lane)) throw new Error("Expected runtime Lane");
|
||||
return Object.assign(lane, { events: harness.events });
|
||||
}
|
||||
|
||||
class BlockingScanStorage extends MemoryStorage {
|
||||
block = false;
|
||||
readonly started = deferred();
|
||||
readonly release = deferred();
|
||||
|
||||
override async scanBranch(query: StorageBranchScan, context: Context) {
|
||||
if (this.block) {
|
||||
this.started.resolve();
|
||||
await this.release.promise;
|
||||
}
|
||||
return super.scanBranch(query, context);
|
||||
}
|
||||
}
|
||||
|
||||
afterEach(async () => {
|
||||
for (const session of sessions.splice(0)) await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
describe("runtime lane watch", () => {
|
||||
it("captures a compaction-bounded transcript and isolates the returned snapshot", async () => {
|
||||
const session = await createSession();
|
||||
await commit(session, [
|
||||
sessionWrites.insertEntry({ id: "root", parentId: null, type: "custom", customType: "root" }),
|
||||
sessionWrites.insertEntry({
|
||||
id: "compact",
|
||||
parentId: "root",
|
||||
type: "compaction",
|
||||
summary: "summary",
|
||||
retainedTail: [],
|
||||
tokensBefore: 10,
|
||||
fromHook: false,
|
||||
}),
|
||||
sessionWrites.insertEntry({
|
||||
id: "after",
|
||||
parentId: "compact",
|
||||
type: "message",
|
||||
message: { role: "user", content: "after", timestamp: 2 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "after"),
|
||||
]);
|
||||
const harness = await attach(session);
|
||||
|
||||
const first = await harness.watch(BACKGROUND_CONTEXT);
|
||||
expect(first.snapshot.transcript.map(({ id }) => id)).toEqual(["compact", "after"]);
|
||||
expect(first.snapshot).toMatchObject({
|
||||
lane: "main",
|
||||
tipId: "after",
|
||||
configuration,
|
||||
stats: { messageCount: 1 },
|
||||
operation: null,
|
||||
queues: [],
|
||||
faulted: false,
|
||||
});
|
||||
first.snapshot.transcript.length = 0;
|
||||
first.snapshot.tipId = null;
|
||||
const second = await harness.watch(BACKGROUND_CONTEXT);
|
||||
expect(second.snapshot.transcript.map(({ id }) => id)).toEqual(["compact", "after"]);
|
||||
expect(second.snapshot.tipId).toBe("after");
|
||||
first.unsubscribe();
|
||||
second.unsubscribe();
|
||||
});
|
||||
|
||||
it("dereferences queues, pending writes, and deferred handles", async () => {
|
||||
const session = await createSession();
|
||||
const handle = { provider: "provider", modelId: "model", api: "test", id: "deferred" };
|
||||
const sourceMessage = fauxAssistantMessage([], { stopReason: "deferred", deferred: handle });
|
||||
const ids = {
|
||||
next: "next",
|
||||
steer: "steer",
|
||||
follow: "follow",
|
||||
write: "write",
|
||||
};
|
||||
const payload = (content: string) => ({
|
||||
type: "message" as const,
|
||||
payload: { role: "user" as const, content, timestamp: 1 },
|
||||
});
|
||||
const operationId = session.idGenerator.next();
|
||||
const state: OperationState = {
|
||||
...runScope({ status: "cancel_requested", requestedAt: 2 }),
|
||||
at: "deferred.suspended",
|
||||
stepId: "step",
|
||||
sourceEntryId: "source",
|
||||
poll: 3,
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
};
|
||||
const meta: OperationMeta = {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
};
|
||||
await commit(session, [
|
||||
sessionWrites.insertEntry({ id: "source", parentId: null, type: "message", message: sourceMessage }),
|
||||
...Object.values(ids).map((id) =>
|
||||
storedValues.setValue(
|
||||
storedValues.pendingEntry(id),
|
||||
id === ids.write ? { type: "custom", customType: "note", payload: { id } } : payload(id),
|
||||
),
|
||||
),
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), meta),
|
||||
storedValues.setValue(storedValues.operationState(operationId), state),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "source"),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [
|
||||
{ entryId: ids.next, kind: "nextRun" },
|
||||
{ entryId: ids.steer, kind: "steer" },
|
||||
{ entryId: ids.follow, kind: "followUp" },
|
||||
{ entryId: ids.write, kind: "write" },
|
||||
],
|
||||
}),
|
||||
]);
|
||||
const harness = await attach(session);
|
||||
|
||||
const watch = await harness.watch(BACKGROUND_CONTEXT);
|
||||
|
||||
expect(watch.snapshot.queues).toMatchObject([
|
||||
{ entryId: ids.next, kind: "nextRun", type: "message", message: { content: ids.next } },
|
||||
{ entryId: ids.steer, kind: "steer", type: "message", message: { content: ids.steer } },
|
||||
{ entryId: ids.follow, kind: "followUp", type: "message", message: { content: ids.follow } },
|
||||
{ entryId: ids.write, kind: "write", type: "custom", customType: "note", data: { id: ids.write } },
|
||||
]);
|
||||
expect(watch.snapshot.operation).toMatchObject({
|
||||
id: operationId,
|
||||
status: "aborting",
|
||||
deferred: { handle, poll: 3 },
|
||||
runningTools: [],
|
||||
});
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("omits streaming presentation when an effect-pending response has no frames", async () => {
|
||||
const session = await createSession();
|
||||
const operationId = session.idGenerator.next();
|
||||
await commit(session, [
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationState(operationId), {
|
||||
...runScope(),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: {
|
||||
stepId: "step",
|
||||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
responseEntryId: "response-without-frames",
|
||||
usageId: "usage",
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1000,
|
||||
}),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
]);
|
||||
const harness = await attach(session);
|
||||
|
||||
const watch = await harness.watch(BACKGROUND_CONTEXT);
|
||||
|
||||
expect(watch.snapshot.operation).not.toHaveProperty("streamingMessage");
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("reduces assistant frames and projects running and settled tools with full-content indexes", async () => {
|
||||
const frameSession = await createSession();
|
||||
const partial = fauxAssistantMessage([], { stopReason: "pending" });
|
||||
const frames: AssistantMessageFrame[] = [
|
||||
{ type: "start", partial },
|
||||
{ type: "text_start", contentIndex: 0, content: { type: "text", text: "" } },
|
||||
{ type: "text_delta", contentIndex: 0, delta: "partial" },
|
||||
];
|
||||
const frameOperationId = frameSession.idGenerator.next();
|
||||
await commit(frameSession, [
|
||||
storedValues.setValue(storedValues.operationMeta(frameOperationId), {
|
||||
operationId: frameOperationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationState(frameOperationId), {
|
||||
...runScope(),
|
||||
at: "assistant.effect_pending",
|
||||
generationContext: {
|
||||
stepId: "step",
|
||||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy: { maxAttempts: 2, baseDelayMs: 0 },
|
||||
overflowRecoveryUsed: false,
|
||||
},
|
||||
attempt: 1,
|
||||
responseEntryId: "response",
|
||||
usageId: "usage",
|
||||
intendedOutputLimit: 100,
|
||||
contextWindow: 1000,
|
||||
}),
|
||||
...frames.map((frame) =>
|
||||
storedValues.appendList(storedValues.pendingAssistantFrames(frameOperationId, "response"), frame),
|
||||
),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: frameOperationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
]);
|
||||
const frameHarness = await attach(frameSession);
|
||||
const frameWatch = await frameHarness.watch(BACKGROUND_CONTEXT);
|
||||
expect(frameWatch.snapshot.operation?.streamingMessage?.content).toEqual([{ type: "text", text: "partial" }]);
|
||||
frameWatch.unsubscribe();
|
||||
|
||||
const toolSession = await createSession();
|
||||
const toolUsage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 3,
|
||||
cacheWrite: 4,
|
||||
totalTokens: 10,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
const assistant = fauxAssistantMessage([
|
||||
{ type: "text", text: "before" },
|
||||
{ type: "toolCall", id: "call-completed", name: "completed", arguments: { source: "completed" } },
|
||||
{ type: "toolCall", id: "call-running", name: "read", arguments: { source: "running" } },
|
||||
{
|
||||
type: "toolCall",
|
||||
id: "call-without-checkpoint",
|
||||
name: "write",
|
||||
arguments: { source: "without-checkpoint" },
|
||||
},
|
||||
{ type: "toolCall", id: "call-ready", name: "real", arguments: { source: "real" } },
|
||||
{ type: "toolCall", id: "call-synthetic", name: "missing", arguments: { source: "synthetic" } },
|
||||
{ type: "toolCall", id: "call-planned", name: "planned", arguments: { source: "planned" } },
|
||||
]);
|
||||
const toolOperationId = toolSession.idGenerator.next();
|
||||
await commit(toolSession, [
|
||||
sessionWrites.insertEntry({ id: "assistant", parentId: null, type: "message", message: assistant }),
|
||||
storedValues.setValue(storedValues.operationMeta(toolOperationId), {
|
||||
operationId: toolOperationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationState(toolOperationId), {
|
||||
...runScope(),
|
||||
at: "tools",
|
||||
batch: {
|
||||
assistantEntryId: "assistant",
|
||||
configuration,
|
||||
turnId: "turn",
|
||||
calls: [
|
||||
{ status: "completed", sourceIndex: 1, resultEntryId: "completed", terminate: false },
|
||||
{ status: "effect_pending", sourceIndex: 2, resultEntryId: "result", replay: "safe" },
|
||||
{
|
||||
status: "effect_pending",
|
||||
sourceIndex: 3,
|
||||
resultEntryId: "without-checkpoint",
|
||||
replay: "never",
|
||||
},
|
||||
{ status: "outcome_ready", sourceIndex: 4, resultEntryId: "ready", terminate: true },
|
||||
{ status: "outcome_ready", sourceIndex: 5, resultEntryId: "synthetic", terminate: false },
|
||||
{ status: "planned", sourceIndex: 6, resultEntryId: "planned" },
|
||||
],
|
||||
},
|
||||
}),
|
||||
sessionWrites.insertEntry({
|
||||
id: "completed",
|
||||
parentId: "assistant",
|
||||
type: "message",
|
||||
message: {
|
||||
role: "toolResult",
|
||||
toolCallId: "call-completed",
|
||||
toolName: "completed",
|
||||
content: [{ type: "text", text: "completed" }],
|
||||
isError: false,
|
||||
timestamp: 2,
|
||||
},
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationToolArgs(toolOperationId, "turn", 2), { path: "file" }),
|
||||
storedValues.setValue(storedValues.operationToolArgs(toolOperationId, "turn", 3), { path: "output" }),
|
||||
storedValues.setValue(storedValues.operationToolArgs(toolOperationId, "turn", 4), { path: "settled" }),
|
||||
storedValues.setValue(storedValues.pendingToolOutput(toolOperationId, "result"), {
|
||||
content: [{ type: "text", text: "partial" }],
|
||||
details: { bytes: 1 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingEntry("ready"), {
|
||||
type: "message",
|
||||
payload: {
|
||||
role: "toolResult",
|
||||
toolCallId: "call-ready",
|
||||
toolName: "real",
|
||||
content: [{ type: "text", text: "settled" }],
|
||||
details: { kind: "real" },
|
||||
usage: toolUsage,
|
||||
addedToolNames: ["later"],
|
||||
isError: false,
|
||||
timestamp: 3,
|
||||
},
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingEntry("synthetic"), {
|
||||
type: "message",
|
||||
payload: {
|
||||
role: "toolResult",
|
||||
toolCallId: "call-synthetic",
|
||||
toolName: "missing",
|
||||
content: [{ type: "text", text: "unavailable" }],
|
||||
isError: true,
|
||||
timestamp: 4,
|
||||
},
|
||||
}),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "completed"),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: toolOperationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
]);
|
||||
const toolHarness = await attach(toolSession);
|
||||
const toolWatch = await toolHarness.watch(BACKGROUND_CONTEXT);
|
||||
expect(toolWatch.snapshot.transcript.map(({ id }) => id)).toEqual(["assistant", "completed"]);
|
||||
expect(toolWatch.snapshot.operation?.runningTools).toEqual([
|
||||
{
|
||||
status: "running",
|
||||
toolCallId: "call-running",
|
||||
toolName: "read",
|
||||
args: { path: "file" },
|
||||
result: { content: [{ type: "text", text: "partial" }], details: { bytes: 1 } },
|
||||
},
|
||||
{
|
||||
status: "running",
|
||||
toolCallId: "call-without-checkpoint",
|
||||
toolName: "write",
|
||||
args: { path: "output" },
|
||||
},
|
||||
{
|
||||
status: "settled",
|
||||
toolCallId: "call-ready",
|
||||
toolName: "real",
|
||||
args: { path: "settled" },
|
||||
result: {
|
||||
content: [{ type: "text", text: "settled" }],
|
||||
details: { kind: "real" },
|
||||
usage: toolUsage,
|
||||
addedToolNames: ["later"],
|
||||
terminate: true,
|
||||
},
|
||||
isError: false,
|
||||
},
|
||||
{
|
||||
status: "settled",
|
||||
toolCallId: "call-synthetic",
|
||||
toolName: "missing",
|
||||
args: { source: "synthetic" },
|
||||
result: {
|
||||
content: [{ type: "text", text: "unavailable" }],
|
||||
details: undefined,
|
||||
},
|
||||
isError: true,
|
||||
},
|
||||
]);
|
||||
expect(toolWatch.snapshot.operation?.runningTools[1]).not.toHaveProperty("result");
|
||||
toolWatch.unsubscribe();
|
||||
});
|
||||
|
||||
it("faults missing or mismatched staged outcome-ready results", async () => {
|
||||
for (const corruption of ["missing", "mismatched"] as const) {
|
||||
const session = await createSession();
|
||||
const operationId = session.idGenerator.next();
|
||||
const writes: Write[] = [
|
||||
sessionWrites.insertEntry({
|
||||
id: "assistant",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: fauxAssistantMessage([
|
||||
{ type: "toolCall", id: "call", name: "read", arguments: { path: "file" } },
|
||||
]),
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationMeta(operationId), {
|
||||
operationId,
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: [] },
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationState(operationId), {
|
||||
...runScope(),
|
||||
at: "tools",
|
||||
batch: {
|
||||
assistantEntryId: "assistant",
|
||||
configuration,
|
||||
turnId: "turn",
|
||||
calls: [{ status: "outcome_ready", sourceIndex: 0, resultEntryId: "result", terminate: false }],
|
||||
},
|
||||
}),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "assistant"),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: operationId,
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
];
|
||||
if (corruption === "mismatched") {
|
||||
writes.push(
|
||||
storedValues.setValue(storedValues.pendingEntry("result"), {
|
||||
type: "message",
|
||||
payload: {
|
||||
role: "toolResult",
|
||||
toolCallId: "other-call",
|
||||
toolName: "read",
|
||||
content: [],
|
||||
isError: false,
|
||||
timestamp: 2,
|
||||
},
|
||||
}),
|
||||
);
|
||||
}
|
||||
await commit(session, writes);
|
||||
const harness = await attach(session);
|
||||
|
||||
await expect(harness.watch(BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(HarnessFault);
|
||||
}
|
||||
});
|
||||
|
||||
it("faults required payload corruption and unsubscribes the incomplete watcher", async () => {
|
||||
const session = await createSession();
|
||||
await commit(session, [
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: null,
|
||||
lastOperationId: null,
|
||||
inbox: [{ entryId: "missing", kind: "nextRun" }],
|
||||
}),
|
||||
]);
|
||||
const harness = await attach(session);
|
||||
let unsubscribed = false;
|
||||
const originalWatch = harness.events.watch.bind(harness.events);
|
||||
vi.spyOn(harness.events, "watch").mockImplementation(
|
||||
<T>(snapshot: T, filter: (event: HarnessEvent) => boolean, context: Context): WatchHandle<T> => {
|
||||
const handle = originalWatch(snapshot, filter, context);
|
||||
return {
|
||||
...handle,
|
||||
unsubscribe: () => {
|
||||
unsubscribed = true;
|
||||
handle.unsubscribe();
|
||||
},
|
||||
};
|
||||
},
|
||||
);
|
||||
|
||||
await expect(harness.watch(BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(HarnessFault);
|
||||
expect(unsubscribed).toBe(true);
|
||||
});
|
||||
|
||||
it("returns snapshot-before plus buffered events when watch wins the lane line", async () => {
|
||||
const storage = new BlockingScanStorage();
|
||||
const session = await createSession(storage);
|
||||
await commit(session, [
|
||||
sessionWrites.insertEntry({ id: "root", parentId: null, type: "custom", customType: "root" }),
|
||||
storedValues.setValue(storedValues.branchTip("main"), "root"),
|
||||
]);
|
||||
const harness = await attach(session);
|
||||
storage.block = true;
|
||||
const watchPromise = harness.watch(BACKGROUND_CONTEXT);
|
||||
await storage.started.promise;
|
||||
const sourceKey = createContextKey<string>("watch.event.source");
|
||||
const sourceContext = withContextValue(sourceKey, "append", BACKGROUND_CONTEXT);
|
||||
const append = harness.appendMessage({ role: "user", content: "later", timestamp: 2 }, sourceContext);
|
||||
storage.release.resolve();
|
||||
const watch = await watchPromise;
|
||||
expect(watch.snapshot.transcript.map(({ id }) => id)).toEqual(["root"]);
|
||||
const seen: Array<{ type: string; sameContext: boolean }> = [];
|
||||
watch.start((event, eventContext) => {
|
||||
seen.push({ type: event.type, sameContext: eventContext === sourceContext });
|
||||
});
|
||||
await append;
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
expect(seen.map(({ type }) => type)).toEqual(["message_start", "message_end", "entry_added"]);
|
||||
expect(seen.every(({ sameContext }) => sameContext)).toBe(true);
|
||||
watch.unsubscribe();
|
||||
});
|
||||
|
||||
it("returns snapshot-after without replay when publication wins", async () => {
|
||||
const session = await createSession();
|
||||
const harness = await attach(session);
|
||||
const entryId = await harness.appendMessage(
|
||||
{ role: "user", content: "existing", timestamp: 1 },
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const watch = await harness.watch(BACKGROUND_CONTEXT);
|
||||
const seen = vi.fn();
|
||||
watch.start(seen);
|
||||
await new Promise((resolve) => setTimeout(resolve, 0));
|
||||
|
||||
expect(watch.snapshot.transcript.map(({ id }) => id)).toEqual([entryId]);
|
||||
expect(seen).not.toHaveBeenCalled();
|
||||
watch.unsubscribe();
|
||||
});
|
||||
});
|
||||
260
third_party/pi/packages/agent/test/harness/session-context.test.ts
vendored
Normal file
260
third_party/pi/packages/agent/test/harness/session-context.test.ts
vendored
Normal file
@@ -0,0 +1,260 @@
|
||||
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { buildSessionContext } from "../../src/harness/session/context.ts";
|
||||
import type {
|
||||
BranchSummaryEntry,
|
||||
CompactionEntry,
|
||||
CustomEntry,
|
||||
MessageEntry,
|
||||
} from "../../src/harness/session/types.ts";
|
||||
import type { AgentMessage } from "../../src/types.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
const usage = {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
|
||||
function userMessage(text: string): AgentMessage {
|
||||
return { role: "user", content: [{ type: "text", text }], timestamp: NOW };
|
||||
}
|
||||
|
||||
function assistantMessage(stopReason: AssistantMessage["stopReason"], text: string): AssistantMessage {
|
||||
return {
|
||||
role: "assistant",
|
||||
content:
|
||||
stopReason === "toolUse"
|
||||
? [{ type: "toolCall", id: "call", name: "read", arguments: {} }]
|
||||
: [{ type: "text", text }],
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-5",
|
||||
usage,
|
||||
stopReason,
|
||||
...(stopReason === "deferred"
|
||||
? { deferred: { provider: "anthropic", modelId: "claude-sonnet-4-5", api: "anthropic-messages", id: "job" } }
|
||||
: {}),
|
||||
timestamp: NOW,
|
||||
};
|
||||
}
|
||||
|
||||
function messageEntry(id: string, parentId: string | null, seq: number, message: AgentMessage): MessageEntry {
|
||||
return { id, parentId, seq, timestamp: NOW, type: "message", message };
|
||||
}
|
||||
|
||||
describe("session context projection", () => {
|
||||
it("filters non-context assistant response entries while preserving valid messages", async () => {
|
||||
const user = userMessage("question");
|
||||
const stopped = assistantMessage("stop", "answer");
|
||||
const length = assistantMessage("length", "truncated answer");
|
||||
const toolUse = assistantMessage("toolUse", "");
|
||||
const failed = assistantMessage("error", "failed");
|
||||
const aborted = assistantMessage("aborted", "aborted");
|
||||
const deferred = assistantMessage("deferred", "");
|
||||
const entries = [
|
||||
messageEntry("user", null, 1, user),
|
||||
messageEntry("failed", "user", 2, failed),
|
||||
messageEntry("stopped", "failed", 3, stopped),
|
||||
messageEntry("aborted", "stopped", 4, aborted),
|
||||
messageEntry("tool-use", "aborted", 5, toolUse),
|
||||
messageEntry("deferred", "tool-use", 6, deferred),
|
||||
messageEntry("length", "deferred", 7, length),
|
||||
];
|
||||
|
||||
expect(await buildSessionContext(entries, undefined, BACKGROUND_CONTEXT)).toEqual([
|
||||
user,
|
||||
stopped,
|
||||
toolUse,
|
||||
length,
|
||||
]);
|
||||
});
|
||||
|
||||
it("projects branch summaries in branch order", async () => {
|
||||
const before = messageEntry("before", null, 1, userMessage("before summary"));
|
||||
const summary: BranchSummaryEntry = {
|
||||
id: "branch-summary",
|
||||
parentId: before.id,
|
||||
seq: 2,
|
||||
timestamp: NOW,
|
||||
type: "branch_summary",
|
||||
fromId: "source-leaf",
|
||||
summary: "work on the abandoned branch",
|
||||
fromHook: false,
|
||||
};
|
||||
const after = messageEntry("after", summary.id, 3, userMessage("after summary"));
|
||||
|
||||
expect(await buildSessionContext([before, summary, after], undefined, BACKGROUND_CONTEXT)).toEqual([
|
||||
userMessage("before summary"),
|
||||
{
|
||||
role: "branchSummary",
|
||||
summary: "work on the abandoned branch",
|
||||
fromId: "source-leaf",
|
||||
timestamp: NOW,
|
||||
},
|
||||
userMessage("after summary"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("filters retained-tail responses without hiding the compaction summary", async () => {
|
||||
const user = userMessage("kept user");
|
||||
const stopped = assistantMessage("stop", "kept answer");
|
||||
const toolUse = assistantMessage("toolUse", "");
|
||||
const length = assistantMessage("length", "kept truncated answer");
|
||||
const failed = assistantMessage("error", "failed");
|
||||
const aborted = assistantMessage("aborted", "aborted");
|
||||
const deferred = assistantMessage("deferred", "");
|
||||
const compaction: CompactionEntry = {
|
||||
id: "compaction",
|
||||
parentId: null,
|
||||
seq: 1,
|
||||
timestamp: NOW,
|
||||
type: "compaction",
|
||||
summary: "summary",
|
||||
retainedTail: [failed, user, aborted, stopped, deferred, toolUse, length],
|
||||
tokensBefore: 100,
|
||||
fromHook: false,
|
||||
};
|
||||
|
||||
expect(await buildSessionContext([compaction], undefined, BACKGROUND_CONTEXT)).toEqual([
|
||||
{ role: "compactionSummary", summary: "summary", tokensBefore: 100, timestamp: NOW },
|
||||
user,
|
||||
stopped,
|
||||
toolUse,
|
||||
length,
|
||||
]);
|
||||
});
|
||||
it("uses only the latest compaction checkpoint and entries after it", async () => {
|
||||
const beforeFirst = messageEntry("before-first", null, 1, userMessage("before first"));
|
||||
const first: CompactionEntry = {
|
||||
id: "first-compaction",
|
||||
parentId: beforeFirst.id,
|
||||
seq: 2,
|
||||
timestamp: NOW,
|
||||
type: "compaction",
|
||||
summary: "stale summary",
|
||||
retainedTail: [userMessage("stale tail")],
|
||||
tokensBefore: 100,
|
||||
fromHook: false,
|
||||
};
|
||||
const between = messageEntry("between", first.id, 3, userMessage("between compactions"));
|
||||
const latest: CompactionEntry = {
|
||||
id: "latest-compaction",
|
||||
parentId: between.id,
|
||||
seq: 4,
|
||||
timestamp: NOW,
|
||||
type: "compaction",
|
||||
summary: "latest summary",
|
||||
retainedTail: [userMessage("latest tail")],
|
||||
tokensBefore: 200,
|
||||
fromHook: false,
|
||||
};
|
||||
const after = messageEntry("after", latest.id, 5, userMessage("after latest"));
|
||||
|
||||
expect(
|
||||
await buildSessionContext([beforeFirst, first, between, latest, after], undefined, BACKGROUND_CONTEXT),
|
||||
).toEqual([
|
||||
{ role: "compactionSummary", summary: "latest summary", tokensBefore: 200, timestamp: NOW },
|
||||
userMessage("latest tail"),
|
||||
userMessage("after latest"),
|
||||
]);
|
||||
});
|
||||
|
||||
it("projects custom entries through synchronous and asynchronous canonical projectors in branch order", async () => {
|
||||
const oldCustom: CustomEntry = {
|
||||
id: "old-custom",
|
||||
parentId: null,
|
||||
seq: 1,
|
||||
timestamp: NOW,
|
||||
type: "custom",
|
||||
customType: "sync",
|
||||
};
|
||||
const compaction: CompactionEntry = {
|
||||
id: "compaction",
|
||||
parentId: oldCustom.id,
|
||||
seq: 2,
|
||||
timestamp: NOW,
|
||||
type: "compaction",
|
||||
summary: "summary",
|
||||
retainedTail: [],
|
||||
tokensBefore: 100,
|
||||
fromHook: false,
|
||||
};
|
||||
const syncCustom: CustomEntry = {
|
||||
id: "sync-custom",
|
||||
parentId: compaction.id,
|
||||
seq: 3,
|
||||
timestamp: NOW,
|
||||
type: "custom",
|
||||
customType: "sync",
|
||||
};
|
||||
const omittedCustom: CustomEntry = {
|
||||
id: "omitted-custom",
|
||||
parentId: syncCustom.id,
|
||||
seq: 4,
|
||||
timestamp: NOW,
|
||||
type: "custom",
|
||||
customType: "omitted",
|
||||
};
|
||||
const asyncCustom: CustomEntry = {
|
||||
id: "async-custom",
|
||||
parentId: omittedCustom.id,
|
||||
seq: 5,
|
||||
timestamp: NOW,
|
||||
type: "custom",
|
||||
customType: "async",
|
||||
};
|
||||
const projectedIds: string[] = [];
|
||||
|
||||
const messages = await buildSessionContext(
|
||||
[oldCustom, compaction, syncCustom, omittedCustom, asyncCustom],
|
||||
{
|
||||
entryProjectors: {
|
||||
sync: (entry) => {
|
||||
projectedIds.push(entry.id);
|
||||
return [userMessage(`projected:${entry.id}`)];
|
||||
},
|
||||
async: async (entry) => {
|
||||
await Promise.resolve();
|
||||
projectedIds.push(entry.id);
|
||||
return [userMessage(`projected:${entry.id}`)];
|
||||
},
|
||||
},
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(projectedIds).toEqual([syncCustom.id, asyncCustom.id]);
|
||||
expect(messages).toEqual([
|
||||
{ role: "compactionSummary", summary: "summary", tokensBefore: 100, timestamp: NOW },
|
||||
userMessage(`projected:${syncCustom.id}`),
|
||||
userMessage(`projected:${asyncCustom.id}`),
|
||||
]);
|
||||
});
|
||||
|
||||
it("propagates custom projector failures", async () => {
|
||||
const custom: CustomEntry = {
|
||||
id: "custom",
|
||||
parentId: null,
|
||||
seq: 1,
|
||||
timestamp: NOW,
|
||||
type: "custom",
|
||||
customType: "broken",
|
||||
};
|
||||
const failure = new Error("projector failed");
|
||||
|
||||
await expect(
|
||||
buildSessionContext(
|
||||
[custom],
|
||||
{
|
||||
entryProjectors: { broken: async () => Promise.reject(failure) },
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toBe(failure);
|
||||
});
|
||||
});
|
||||
77
third_party/pi/packages/agent/test/harness/session-create-branch.test.ts
vendored
Normal file
77
third_party/pi/packages/agent/test/harness/session-create-branch.test.ts
vendored
Normal file
@@ -0,0 +1,77 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { insertEntry } from "../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import {
|
||||
SessionBranchExistsError,
|
||||
SessionInvalidBranchError,
|
||||
SessionUnknownTargetError,
|
||||
StorageBackedSession,
|
||||
} from "../../src/harness/session/session.ts";
|
||||
import { branchTip, laneConfig, laneState } from "../../src/harness/session/values.ts";
|
||||
|
||||
function createSession(): StorageBackedSession {
|
||||
return new StorageBackedSession(
|
||||
{ id: "session", createdAt: 1, storageVersion: 1 },
|
||||
new MemoryStorage({ now: () => 10 }),
|
||||
);
|
||||
}
|
||||
|
||||
describe("Session.createBranch", () => {
|
||||
it("creates only the data Branch at a validated target", async () => {
|
||||
const session = createSession();
|
||||
await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
insertEntry({
|
||||
id: "target",
|
||||
parentId: null,
|
||||
type: "custom",
|
||||
customType: "target",
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const branch = await session.createBranch("main", "target", BACKGROUND_CONTEXT);
|
||||
expect(await branch.getTipId(BACKGROUND_CONTEXT)).toBe("target");
|
||||
expect((await session.getValue(branchTip("main"), BACKGROUND_CONTEXT))?.value).toBe("target");
|
||||
expect(await session.getValue(laneConfig("main"), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
expect(await session.getValue(laneState("main"), BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("validates names and non-null targets", async () => {
|
||||
const session = createSession();
|
||||
await expect(session.createBranch("", null, BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(
|
||||
SessionInvalidBranchError,
|
||||
);
|
||||
await expect(session.createBranch("bad\u0000name", null, BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(
|
||||
SessionInvalidBranchError,
|
||||
);
|
||||
await expect(session.createBranch("main", "missing", BACKGROUND_CONTEXT)).rejects.toBeInstanceOf(
|
||||
SessionUnknownTargetError,
|
||||
);
|
||||
expect(await session.branch("main", BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects duplicates atomically, including concurrent creation", async () => {
|
||||
const session = createSession();
|
||||
const results = await Promise.allSettled([
|
||||
session.createBranch("main", null, BACKGROUND_CONTEXT),
|
||||
session.createBranch("main", null, BACKGROUND_CONTEXT),
|
||||
]);
|
||||
expect(results.filter(({ status }) => status === "fulfilled")).toHaveLength(1);
|
||||
const rejected = results.find(({ status }) => status === "rejected");
|
||||
expect(rejected).toMatchObject({
|
||||
status: "rejected",
|
||||
reason: expect.any(SessionBranchExistsError),
|
||||
});
|
||||
expect(await session.branch("main", BACKGROUND_CONTEXT)).toBeDefined();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
20
third_party/pi/packages/agent/test/harness/session-test-utils.ts
vendored
Normal file
20
third_party/pi/packages/agent/test/harness/session-test-utils.ts
vendored
Normal file
@@ -0,0 +1,20 @@
|
||||
import { existsSync, mkdirSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { afterEach } from "vitest";
|
||||
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
export function createTempDir(): string {
|
||||
const dir = join(tmpdir(), `pi-agent-session-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(dir, { recursive: true });
|
||||
tempDirs.push(dir);
|
||||
return dir;
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
while (tempDirs.length > 0) {
|
||||
const dir = tempDirs.pop()!;
|
||||
if (existsSync(dir)) rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
150
third_party/pi/packages/agent/test/harness/skills.test.ts
vendored
Normal file
150
third_party/pi/packages/agent/test/harness/skills.test.ts
vendored
Normal file
@@ -0,0 +1,150 @@
|
||||
import { symlink } from "node:fs/promises";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import { loadSkills, loadSourcedSkills } from "../../src/harness/skills.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
describe("loadSkills", () => {
|
||||
it("loads SKILL.md files through the execution environment", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir(".agents/skills/example", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile(
|
||||
".agents/skills/example/SKILL.md",
|
||||
`---
|
||||
name: example
|
||||
description: Example skill
|
||||
disable-model-invocation: true
|
||||
---
|
||||
Use this skill.
|
||||
`,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const { skills, diagnostics } = await loadSkills(env, ".agents/skills", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(diagnostics).toEqual([]);
|
||||
expect(skills).toEqual([
|
||||
{
|
||||
name: "example",
|
||||
description: "Example skill",
|
||||
content: "Use this skill.",
|
||||
filePath: join(root, ".agents/skills/example/SKILL.md"),
|
||||
disableModelInvocation: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("loads skills through symlinked directories", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("actual/example", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile(
|
||||
"actual/example/SKILL.md",
|
||||
"---\nname: example\ndescription: Example skill\n---\nUse this skill.",
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await symlink(join(root, "actual"), join(root, "skills-link"));
|
||||
|
||||
const { skills } = await loadSkills(env, "skills-link", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(skills.map((skill) => skill.name)).toEqual(["example"]);
|
||||
expect(skills[0]?.filePath).toBe(join(root, "skills-link/example/SKILL.md"));
|
||||
});
|
||||
|
||||
it("preserves source info for sourced skills", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("user/example", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile(
|
||||
"user/example/SKILL.md",
|
||||
"---\nname: example\ndescription: Example skill\n---\nUse this skill.",
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const { skills, diagnostics } = await loadSourcedSkills(
|
||||
env,
|
||||
[{ path: "user", source: { type: "user" as const } }],
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(diagnostics).toEqual([]);
|
||||
expect(skills).toEqual([
|
||||
{
|
||||
skill: {
|
||||
name: "example",
|
||||
description: "Example skill",
|
||||
content: "Use this skill.",
|
||||
filePath: join(root, "user/example/SKILL.md"),
|
||||
disableModelInvocation: false,
|
||||
},
|
||||
source: { type: "user" },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("attaches source info to diagnostics", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("user/broken", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile("user/broken/SKILL.md", "---\nname: broken\n---\nMissing description.", BACKGROUND_CONTEXT);
|
||||
|
||||
const { skills, diagnostics } = await loadSourcedSkills(
|
||||
env,
|
||||
[{ path: "user", source: { type: "user" as const } }],
|
||||
undefined,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(skills).toEqual([]);
|
||||
expect(diagnostics).toEqual([
|
||||
{
|
||||
type: "warning",
|
||||
code: "invalid_metadata",
|
||||
message: "description is required",
|
||||
path: join(root, "user/broken/SKILL.md"),
|
||||
source: { type: "user" },
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it("loads direct markdown children only from the root directory", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("skills/nested", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile("skills/root.md", "---\ndescription: Root skill\n---\nRoot content", BACKGROUND_CONTEXT);
|
||||
await env.writeFile(
|
||||
"skills/nested/ignored.md",
|
||||
"---\ndescription: Ignored\n---\nIgnored content",
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const { skills } = await loadSkills(env, "skills", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(skills.map((skill) => skill.name)).toEqual(["skills"]);
|
||||
expect(skills[0]?.content).toBe("Root content");
|
||||
});
|
||||
|
||||
it("ignores root markdown docs that do not declare skills", async () => {
|
||||
const root = createTempDir();
|
||||
const env = new NodeExecutionEnv({ cwd: root });
|
||||
await env.createDir("skills/nested-skill", { recursive: true }, BACKGROUND_CONTEXT);
|
||||
await env.writeFile("skills/README.md", "# Shared skills\n\nDocumentation.", BACKGROUND_CONTEXT);
|
||||
await env.writeFile("skills/AGENTS.md", "# Agent notes\n\nDocumentation.", BACKGROUND_CONTEXT);
|
||||
await env.writeFile("skills/CLAUDE.md", "---\ndescription: [invalid\n---\n\nDocumentation.", BACKGROUND_CONTEXT);
|
||||
await env.writeFile("skills/root.md", "---\ndescription: Root skill\n---\nRoot content", BACKGROUND_CONTEXT);
|
||||
await env.writeFile(
|
||||
"skills/nested-skill/SKILL.md",
|
||||
"---\nname: nested-skill\ndescription: Nested skill\n---\nNested content",
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
const { skills, diagnostics } = await loadSkills(env, "skills", BACKGROUND_CONTEXT);
|
||||
|
||||
expect(diagnostics).toEqual([]);
|
||||
expect(skills.map((skill) => skill.name).sort()).toEqual(["nested-skill", "skills"]);
|
||||
});
|
||||
});
|
||||
397
third_party/pi/packages/agent/test/harness/storage-backed-session.test.ts
vendored
Normal file
397
third_party/pi/packages/agent/test/harness/storage-backed-session.test.ts
vendored
Normal file
@@ -0,0 +1,397 @@
|
||||
import type { AssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import type { CustomMessage } from "../../src/harness/messages.ts";
|
||||
import * as sessionWrites from "../../src/harness/session/commit.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import { StorageBackedSession } from "../../src/harness/session/session.ts";
|
||||
import { GatingStorage, InstrumentedStorage } from "../../src/harness/session/testing/index.ts";
|
||||
import type {
|
||||
MessageEntry,
|
||||
NewEntry,
|
||||
Session,
|
||||
SessionMetadata,
|
||||
SessionMutator,
|
||||
Write,
|
||||
} from "../../src/harness/session/types.ts";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
import { deferred } from "./runtime/test-utils.ts";
|
||||
|
||||
const NOW = 1_700_000_000_000;
|
||||
const ENTRY_ID = "00000000-0000-7000-8000-000000000001";
|
||||
const metadata = {
|
||||
id: "session",
|
||||
createdAt: NOW,
|
||||
storageVersion: 1,
|
||||
cwd: "/workspace",
|
||||
} satisfies SessionMetadata;
|
||||
|
||||
function commitSession(session: Session, transaction: Write[]) {
|
||||
return session.mutate((mutator) => mutator.commit(transaction, BACKGROUND_CONTEXT), BACKGROUND_CONTEXT);
|
||||
}
|
||||
|
||||
describe("StorageBackedSession", () => {
|
||||
it("delegates typed values directly without validation or cloning", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const data = { nested: ["original"] };
|
||||
const transaction: Write[] = [
|
||||
sessionWrites.insertEntry({ id: ENTRY_ID, parentId: null, type: "custom", customType: "note", data }),
|
||||
storedValues.setValue(storedValues.value<unknown>("test.value", "state"), data),
|
||||
];
|
||||
|
||||
const result = await commitSession(session, transaction);
|
||||
|
||||
expect(storage.getCommitAttempts()[0]).toBe(transaction);
|
||||
const entry = (await session.getEntries([ENTRY_ID], BACKGROUND_CONTEXT)).get(ENTRY_ID);
|
||||
expect(entry).toMatchObject({ seq: result.seqs[0], timestamp: NOW });
|
||||
if (entry?.type !== "custom") throw new Error("Expected custom entry");
|
||||
expect(entry.data).toBe(data);
|
||||
expect(
|
||||
(await session.getValue(storedValues.value<unknown>("test.value", "state"), BACKGROUND_CONTEXT))?.value,
|
||||
).toBe(data);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("composes bound values and lists atomically with entries and usage", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const scalar = storedValues.value<string>("test.application.scalar");
|
||||
const events = storedValues.list<string>("test.application.events");
|
||||
|
||||
const result = await session.mutate(
|
||||
(mutator) =>
|
||||
mutator.commit(
|
||||
[
|
||||
sessionWrites.insertEntry({ id: ENTRY_ID, parentId: null, type: "custom", customType: "note" }),
|
||||
storedValues.setValue(scalar, "state"),
|
||||
storedValues.appendList(events, "event"),
|
||||
sessionWrites.insertUsage({
|
||||
id: "usage",
|
||||
adjustment: false,
|
||||
usage: {
|
||||
input: 1,
|
||||
output: 1,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 2,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
}),
|
||||
],
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(result.seqs).toHaveLength(4);
|
||||
expect((await session.getValue(scalar, BACKGROUND_CONTEXT))?.seq).toBe(result.seqs[1]);
|
||||
expect(await session.readList(events, undefined, BACKGROUND_CONTEXT)).toEqual([
|
||||
{ seq: result.seqs[2], value: "event" },
|
||||
]);
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("serializes read-modify-write callbacks on the single Session line", async () => {
|
||||
const session = new StorageBackedSession(metadata, new MemoryStorage({ now: () => NOW }));
|
||||
const counter = storedValues.value<number>("test.counter");
|
||||
const increment = () =>
|
||||
session.mutate(async (mutator) => {
|
||||
const next = ((await mutator.getValue(counter, BACKGROUND_CONTEXT))?.value ?? 0) + 1;
|
||||
await mutator.commit([storedValues.setValue(counter, next)], BACKGROUND_CONTEXT);
|
||||
return next;
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
await expect(Promise.all([increment(), increment()])).resolves.toEqual([1, 2]);
|
||||
expect((await session.getValue(counter, BACKGROUND_CONTEXT))?.value).toBe(2);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("keeps separate direct reads and writes deliberately non-atomic", async () => {
|
||||
const session = new StorageBackedSession(metadata, new MemoryStorage({ now: () => NOW }));
|
||||
const counter = storedValues.value<number>("test.counter");
|
||||
const bothRead = deferred();
|
||||
let readers = 0;
|
||||
const increment = async () => {
|
||||
const next = ((await session.getValue(counter, BACKGROUND_CONTEXT))?.value ?? 0) + 1;
|
||||
readers++;
|
||||
if (readers === 2) bothRead.resolve();
|
||||
await bothRead.promise;
|
||||
await session.setValue(counter, next, BACKGROUND_CONTEXT);
|
||||
};
|
||||
|
||||
await Promise.all([increment(), increment()]);
|
||||
expect((await session.getValue(counter, BACKGROUND_CONTEXT))?.value).toBe(1);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("queues a nested public writer until its owning callback returns", async () => {
|
||||
const session = new StorageBackedSession(metadata, new MemoryStorage({ now: () => NOW }));
|
||||
let nested: Promise<void> | undefined;
|
||||
await session.mutate(async () => {
|
||||
nested = session.setValue(storedValues.sessionName, "nested", BACKGROUND_CONTEXT);
|
||||
let settled = false;
|
||||
void nested.finally(() => {
|
||||
settled = true;
|
||||
});
|
||||
await Promise.resolve();
|
||||
expect(settled).toBe(false);
|
||||
}, BACKGROUND_CONTEXT);
|
||||
if (nested === undefined) throw new Error("Expected nested writer");
|
||||
await nested;
|
||||
expect(await session.getName(BACKGROUND_CONTEXT)).toBe("nested");
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("exposes either side of an atomic multi-write commit to direct reads", async () => {
|
||||
const first = storedValues.value<string>("test.atomic", "first");
|
||||
const second = storedValues.value<string>("test.atomic", "second");
|
||||
const storage = new GatingStorage(new MemoryStorage({ now: () => NOW }));
|
||||
await storage.commit(
|
||||
[storedValues.setValue(first, "old"), storedValues.setValue(second, "old")],
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
storage.arm();
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const committing = commitSession(session, [
|
||||
storedValues.setValue(first, "new"),
|
||||
storedValues.setValue(second, "new"),
|
||||
]);
|
||||
await storage.waitPending();
|
||||
|
||||
expect((await session.getValue(first, BACKGROUND_CONTEXT))?.value).toBe("old");
|
||||
expect((await session.getValue(second, BACKGROUND_CONTEXT))?.value).toBe("old");
|
||||
await storage.next();
|
||||
await committing;
|
||||
expect((await session.getValue(first, BACKGROUND_CONTEXT))?.value).toBe("new");
|
||||
expect((await session.getValue(second, BACKGROUND_CONTEXT))?.value).toBe("new");
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("holds the explicit Session barrier through commit until end", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const mutation = await session.beginMutation(BACKGROUND_CONTEXT);
|
||||
let queuedStarted = false;
|
||||
const queued = session.mutate(() => {
|
||||
queuedStarted = true;
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
await Promise.resolve();
|
||||
expect(queuedStarted).toBe(false);
|
||||
expect(await mutation.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
const result = await mutation.commit([], BACKGROUND_CONTEXT);
|
||||
expect(result.seqs).toEqual([]);
|
||||
expect(queuedStarted).toBe(false);
|
||||
expect(await mutation.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await mutation.end(BACKGROUND_CONTEXT);
|
||||
await queued;
|
||||
expect(queuedStarted).toBe(true);
|
||||
expect(storage.getCommitAttempts()).toEqual([[]]);
|
||||
expect(() => mutation.getEntries([], BACKGROUND_CONTEXT)).toThrow("outside its mutation callback");
|
||||
expect(() => mutation.commit([], BACKGROUND_CONTEXT)).toThrow("outside its mutation callback");
|
||||
await expect(mutation.end(BACKGROUND_CONTEXT)).resolves.toBeUndefined();
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("allows direct reads to observe a committed value before the mutation scope ends", async () => {
|
||||
const session = new StorageBackedSession(metadata, new MemoryStorage({ now: () => NOW }));
|
||||
const mutation = await session.beginMutation(BACKGROUND_CONTEXT);
|
||||
expect(await session.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await mutation.commit([storedValues.setValue(storedValues.sessionName, "visible")], BACKGROUND_CONTEXT);
|
||||
let queuedStarted = false;
|
||||
const queued = session.mutate(() => {
|
||||
queuedStarted = true;
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
expect((await session.getValue(storedValues.sessionName, BACKGROUND_CONTEXT))?.value).toBe("visible");
|
||||
expect(queuedStarted).toBe(false);
|
||||
await mutation.end(BACKGROUND_CONTEXT);
|
||||
await queued;
|
||||
expect(queuedStarted).toBe(true);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("ends an explicit mutation without committing and lets close finish", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const mutation = await session.beginMutation(BACKGROUND_CONTEXT);
|
||||
let closed = false;
|
||||
const closing = session.close(BACKGROUND_CONTEXT).then(() => {
|
||||
closed = true;
|
||||
});
|
||||
|
||||
await Promise.resolve();
|
||||
expect(closed).toBe(false);
|
||||
await mutation.end(BACKGROUND_CONTEXT);
|
||||
await closing;
|
||||
expect(closed).toBe(true);
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
});
|
||||
|
||||
it("exposes explicit branch scans through the Session and callback-scoped mutator", async () => {
|
||||
const storage = new MemoryStorage({ now: () => NOW });
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const childId = "00000000-0000-7000-8000-000000000002";
|
||||
await commitSession(session, [
|
||||
sessionWrites.insertEntry({ id: ENTRY_ID, parentId: null, type: "custom", customType: "root" }),
|
||||
sessionWrites.insertEntry({ id: childId, parentId: ENTRY_ID, type: "custom", customType: "child" }),
|
||||
]);
|
||||
|
||||
await expect(
|
||||
session.scanBranch({ start: childId, order: "oldestFirst" }, BACKGROUND_CONTEXT),
|
||||
).resolves.toMatchObject([{ id: ENTRY_ID }, { id: childId }]);
|
||||
let captured: SessionMutator | undefined;
|
||||
await session.mutate(async (mutator) => {
|
||||
captured = mutator;
|
||||
await expect(mutator.scanBranch({ start: childId, limit: 1 }, BACKGROUND_CONTEXT)).resolves.toMatchObject([
|
||||
{ id: childId },
|
||||
]);
|
||||
}, BACKGROUND_CONTEXT);
|
||||
const invalidated = captured;
|
||||
if (invalidated === undefined) throw new Error("Expected captured mutator");
|
||||
expect(() => invalidated.scanBranch({ start: childId }, BACKGROUND_CONTEXT)).toThrow(
|
||||
"outside its mutation callback",
|
||||
);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("rejects pending assistant entries at the durable session write boundary", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const pending: AssistantMessage = {
|
||||
role: "assistant",
|
||||
content: [],
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
model: "claude-sonnet-4-5",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "pending",
|
||||
timestamp: NOW,
|
||||
};
|
||||
|
||||
await expect(
|
||||
commitSession(session, [
|
||||
sessionWrites.insertEntry({ id: ENTRY_ID, parentId: null, type: "message", message: pending }),
|
||||
]),
|
||||
).rejects.toThrow("Cannot persist a pending assistant message");
|
||||
expect(storage.getCommitAttempts()).toEqual([]);
|
||||
expect(await session.getEntries([ENTRY_ID], BACKGROUND_CONTEXT)).toEqual(new Map());
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("trusts typed custom messages without repository schema registration", async () => {
|
||||
const storage = new MemoryStorage({ now: () => NOW });
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const message: CustomMessage = {
|
||||
role: "custom",
|
||||
customType: "notice",
|
||||
content: "maintenance",
|
||||
display: true,
|
||||
timestamp: NOW,
|
||||
};
|
||||
const entry: NewEntry<MessageEntry> = { id: ENTRY_ID, parentId: null, type: "message", message };
|
||||
|
||||
const result = await commitSession(session, [sessionWrites.insertEntry(entry)]);
|
||||
|
||||
expect((await session.getEntries([ENTRY_ID], BACKGROUND_CONTEXT)).get(ENTRY_ID)).toEqual({
|
||||
...entry,
|
||||
seq: result.firstSeq,
|
||||
timestamp: result.timestamp,
|
||||
});
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("serializes mutations, permits one commit attempt, and invalidates the mutator", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
let captured: SessionMutator | undefined;
|
||||
|
||||
await session.mutate(async (mutator) => {
|
||||
captured = mutator;
|
||||
expect(await mutator.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).toBeUndefined();
|
||||
await mutator.commit([storedValues.setValue(storedValues.sessionName, "committed")], BACKGROUND_CONTEXT);
|
||||
await expect(mutator.commit([], BACKGROUND_CONTEXT)).rejects.toThrow("commit already attempted");
|
||||
}, BACKGROUND_CONTEXT);
|
||||
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
expect(await session.getName(BACKGROUND_CONTEXT)).toBe("committed");
|
||||
const invalidated = captured;
|
||||
if (invalidated === undefined) throw new Error("Expected captured mutator");
|
||||
expect(() => invalidated.getEntries([], BACKGROUND_CONTEXT)).toThrow("outside its mutation callback");
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("consumes the commit guard when the first commit fails", async () => {
|
||||
const storage = new InstrumentedStorage(new MemoryStorage({ now: () => NOW }));
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
const transaction = [
|
||||
sessionWrites.insertEntry({ id: ENTRY_ID, parentId: "missing", type: "custom", customType: "note" }),
|
||||
] satisfies Write[];
|
||||
|
||||
await expect(
|
||||
session.mutate(async (mutator) => {
|
||||
await expect(mutator.commit(transaction, BACKGROUND_CONTEXT)).rejects.toThrow("Missing parent entry");
|
||||
await expect(mutator.commit([], BACKGROUND_CONTEXT)).rejects.toThrow("commit already attempted");
|
||||
}, BACKGROUND_CONTEXT),
|
||||
).resolves.toBeUndefined();
|
||||
expect(storage.getCommitAttempts()).toHaveLength(1);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("mints distinct follower ids with the leader timestamp", async () => {
|
||||
const session = new StorageBackedSession(metadata, new MemoryStorage({ now: () => NOW }));
|
||||
const leaderTimestamp = 0x0123456789ab;
|
||||
const leader = session.idGenerator.next(leaderTimestamp);
|
||||
const followers = [session.idGenerator.next(leaderTimestamp), session.idGenerator.next(leaderTimestamp)];
|
||||
const decodeTimestamp = (id: string): number => Number.parseInt(id.replaceAll("-", "").slice(0, 12), 16);
|
||||
|
||||
expect([leader, ...followers].map(decodeTimestamp)).toEqual([leaderTimestamp, leaderTimestamp, leaderTimestamp]);
|
||||
expect(new Set([leader, ...followers])).toHaveLength(3);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("accepts an injected id generator for deterministic execution tests", async () => {
|
||||
let next = 0;
|
||||
const idGenerator = { next: (timestampMs?: number) => `${timestampMs ?? "now"}:${++next}` };
|
||||
const session = new StorageBackedSession(metadata, new MemoryStorage({ now: () => NOW }), { idGenerator });
|
||||
|
||||
expect(session.idGenerator).toBe(idGenerator);
|
||||
expect(session.idGenerator.next(7)).toBe("7:1");
|
||||
expect(session.idGenerator.next()).toBe("now:2");
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("exposes metadata directly and the shared UUIDv7 id generator", async () => {
|
||||
const sourceMetadata = { ...metadata };
|
||||
const session = new StorageBackedSession(sourceMetadata, new MemoryStorage({ now: () => NOW }));
|
||||
|
||||
expect(session.metadata).toBe(sourceMetadata);
|
||||
expect(session.idGenerator.next()).toMatch(
|
||||
/^[0-9a-f]{8}-[0-9a-f]{4}-7[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/,
|
||||
);
|
||||
await session.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("closes idempotently and rejects operations not admitted before close", async () => {
|
||||
const storage = new MemoryStorage({ now: () => NOW });
|
||||
const session = new StorageBackedSession(metadata, storage);
|
||||
|
||||
await Promise.all([session.close(BACKGROUND_CONTEXT), session.close(BACKGROUND_CONTEXT)]);
|
||||
await expect(session.mutate(() => undefined, BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
await expect(session.getEntries([], BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
await expect(session.getValue(storedValues.sessionName, BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
await expect(session.scanValues(storedValues.sessionName, BACKGROUND_CONTEXT)).rejects.toThrow(
|
||||
"Session is closed",
|
||||
);
|
||||
await expect(session.scanBranch({ start: ENTRY_ID }, BACKGROUND_CONTEXT)).rejects.toThrow("Session is closed");
|
||||
});
|
||||
});
|
||||
66
third_party/pi/packages/agent/test/harness/system-prompt.test.ts
vendored
Normal file
66
third_party/pi/packages/agent/test/harness/system-prompt.test.ts
vendored
Normal file
@@ -0,0 +1,66 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { formatSkillsForSystemPrompt } from "../../src/harness/system-prompt.ts";
|
||||
|
||||
const visibleSkill = {
|
||||
name: "visible",
|
||||
description: "Use <this> & that",
|
||||
content: "visible content",
|
||||
filePath: "/skills/visible/SKILL.md",
|
||||
};
|
||||
|
||||
const secondSkill = {
|
||||
name: "second",
|
||||
description: "Second skill",
|
||||
content: "second content",
|
||||
filePath: "/skills/second/SKILL.md",
|
||||
};
|
||||
|
||||
const disabledSkill = {
|
||||
name: "hidden",
|
||||
description: "Hidden",
|
||||
content: "hidden content",
|
||||
filePath: "/skills/hidden/SKILL.md",
|
||||
disableModelInvocation: true,
|
||||
};
|
||||
|
||||
describe("formatSkillsForSystemPrompt", () => {
|
||||
it("formats visible skills in order and skips model-disabled skills", () => {
|
||||
expect(formatSkillsForSystemPrompt([visibleSkill, disabledSkill, secondSkill])).toBe(
|
||||
`The following skills provide specialized instructions for specific tasks.
|
||||
Read the full skill file when the task matches its description.
|
||||
When a skill file references a relative path, resolve it against the skill directory (parent of SKILL.md / dirname of the path) and use that absolute path in tool commands.
|
||||
|
||||
<available_skills>
|
||||
<skill>
|
||||
<name>visible</name>
|
||||
<description>Use <this> & that</description>
|
||||
<location>/skills/visible/SKILL.md</location>
|
||||
</skill>
|
||||
<skill>
|
||||
<name>second</name>
|
||||
<description>Second skill</description>
|
||||
<location>/skills/second/SKILL.md</location>
|
||||
</skill>
|
||||
</available_skills>`,
|
||||
);
|
||||
});
|
||||
|
||||
it("returns an empty string when no skills are model-visible", () => {
|
||||
expect(formatSkillsForSystemPrompt([disabledSkill])).toBe("");
|
||||
});
|
||||
|
||||
it("escapes XML in all model-visible skill fields", () => {
|
||||
expect(
|
||||
formatSkillsForSystemPrompt([
|
||||
{
|
||||
name: "a&b",
|
||||
description: `Quote "double" and 'single'`,
|
||||
content: "content",
|
||||
filePath: '/skills/<bad>&"quote"/SKILL.md',
|
||||
},
|
||||
]),
|
||||
).toContain(
|
||||
"<name>a&b</name>\n <description>Quote "double" and 'single'</description>\n <location>/skills/<bad>&"quote"/SKILL.md</location>",
|
||||
);
|
||||
});
|
||||
});
|
||||
204
third_party/pi/packages/agent/test/harness/telemetry.test.ts
vendored
Normal file
204
third_party/pi/packages/agent/test/harness/telemetry.test.ts
vendored
Normal file
@@ -0,0 +1,204 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import { createTypedSpanStarter, NOOP_TELEMETRY_CONTEXT, type TelemetryContext } from "@earendil-works/pi-telemetry";
|
||||
import { describe, expect, expectTypeOf, it } from "vitest";
|
||||
import { renderAgentTelemetrySchemaMarkdown } from "../../scripts/generate-telemetry-docs.ts";
|
||||
import { BACKGROUND_CONTEXT, withTelemetryContext } from "../../src/harness/context.ts";
|
||||
import {
|
||||
AGENT_TELEMETRY_SCHEMAS,
|
||||
AI_TELEMETRY_SCHEMA,
|
||||
type AiSpanEndAttributes,
|
||||
type AiSpanStartAttributes,
|
||||
HARNESS_TELEMETRY_SCHEMA,
|
||||
type HarnessSpanEndAttributes,
|
||||
type HarnessSpanStartAttributes,
|
||||
startAiSpan,
|
||||
startHarnessSpan,
|
||||
} from "../../src/harness/telemetry.ts";
|
||||
|
||||
describe("agent telemetry schemas", () => {
|
||||
it("serializes both schemas and generates the checked-in reference", () => {
|
||||
expect(() => JSON.stringify(AI_TELEMETRY_SCHEMA)).not.toThrow();
|
||||
expect(() => JSON.stringify(HARNESS_TELEMETRY_SCHEMA)).not.toThrow();
|
||||
expect(AGENT_TELEMETRY_SCHEMAS).toEqual([AI_TELEMETRY_SCHEMA, HARNESS_TELEMETRY_SCHEMA]);
|
||||
expect(Object.keys(HARNESS_TELEMETRY_SCHEMA.spans)).toEqual([
|
||||
"pi.harness.run",
|
||||
"pi.harness.compaction",
|
||||
"pi.harness.navigation",
|
||||
"pi.harness.checkpoint",
|
||||
"pi.harness.turn",
|
||||
"pi.harness.step",
|
||||
"pi.harness.tool",
|
||||
"pi.harness.hook",
|
||||
"pi.harness.sleep",
|
||||
"pi.harness.event_handler",
|
||||
"pi.session.write",
|
||||
]);
|
||||
const actual = readFileSync(resolve(import.meta.dirname, "../../docs/telemetry-schema.md"), "utf8");
|
||||
expect(actual).toBe(renderAgentTelemetrySchemaMarkdown());
|
||||
});
|
||||
|
||||
it("starts AI-request and harness spans through one composed typed starter", async () => {
|
||||
const startSpan = createTypedSpanStarter(NOOP_TELEMETRY_CONTEXT, AGENT_TELEMETRY_SCHEMAS);
|
||||
await startSpan(
|
||||
"pi.harness.step",
|
||||
{
|
||||
"pi.lane.name": "main",
|
||||
"pi.operation.id": "operation",
|
||||
"pi.step.kind": "assistant",
|
||||
"pi.step.attempt": 1,
|
||||
},
|
||||
async (stepSpan, startChildSpan) => {
|
||||
stepSpan.setAttributes({ "pi.step.outcome": "succeeded" });
|
||||
await startChildSpan(
|
||||
"pi.ai.request",
|
||||
{
|
||||
"pi.ai.operation": "stream",
|
||||
"pi.ai.provider": "provider",
|
||||
"pi.ai.model": "model",
|
||||
"pi.ai.api": "api",
|
||||
"pi.ai.streaming": true,
|
||||
},
|
||||
(requestSpan) => {
|
||||
requestSpan.setAttributes({ "pi.ai.response.stop_reason": "stop" });
|
||||
},
|
||||
);
|
||||
},
|
||||
);
|
||||
});
|
||||
|
||||
it("infers exact AI start and optional end attributes", async () => {
|
||||
type Start = AiSpanStartAttributes<"pi.ai.request">;
|
||||
type End = AiSpanEndAttributes<"pi.ai.request">;
|
||||
expectTypeOf<Start>().toMatchTypeOf<{
|
||||
"pi.ai.operation": "stream" | "fetch_deferred" | "cancel_deferred" | "generate_images";
|
||||
"pi.ai.provider": string;
|
||||
"pi.ai.model": string;
|
||||
"pi.ai.api": string;
|
||||
"pi.ai.streaming": boolean;
|
||||
"pi.ai.deferred"?: boolean;
|
||||
}>();
|
||||
expectTypeOf<End["pi.ai.response.stop_reason"]>().toEqualTypeOf<
|
||||
"stop" | "length" | "tool_use" | "error" | "aborted" | "deferred" | undefined
|
||||
>();
|
||||
|
||||
const telemetryContext: TelemetryContext = NOOP_TELEMETRY_CONTEXT;
|
||||
const context = withTelemetryContext(telemetryContext, BACKGROUND_CONTEXT);
|
||||
await startAiSpan(
|
||||
"pi.ai.request",
|
||||
{
|
||||
"pi.ai.operation": "stream",
|
||||
"pi.ai.provider": "provider",
|
||||
"pi.ai.model": "model",
|
||||
"pi.ai.api": "api",
|
||||
"pi.ai.streaming": true,
|
||||
},
|
||||
(span) => {
|
||||
span.setAttributes({ "pi.ai.response.stop_reason": "tool_use" });
|
||||
// @ts-expect-error pi.ai.request declares no span events
|
||||
span.addEvent("chunk");
|
||||
},
|
||||
context,
|
||||
);
|
||||
|
||||
const compileTimeFailures = () => {
|
||||
const extraAttributes = {
|
||||
"pi.ai.operation": "stream",
|
||||
"pi.ai.provider": "provider",
|
||||
"pi.ai.model": "model",
|
||||
"pi.ai.api": "api",
|
||||
"pi.ai.streaming": true,
|
||||
"pi.ai.unknown": true,
|
||||
} as const;
|
||||
// @ts-expect-error variables with unknown attributes are rejected
|
||||
void startAiSpan("pi.ai.request", extraAttributes, () => {}, context);
|
||||
// @ts-expect-error missing required start attributes
|
||||
void startAiSpan("pi.ai.request", { "pi.ai.operation": "stream" }, () => {}, context);
|
||||
};
|
||||
expectTypeOf(compileTimeFailures).toBeFunction();
|
||||
});
|
||||
|
||||
it("infers per-span harness literals and optional completion enrichment", async () => {
|
||||
type RunStart = HarnessSpanStartAttributes<"pi.harness.run">;
|
||||
type RunEnd = HarnessSpanEndAttributes<"pi.harness.run">;
|
||||
type WriteStart = HarnessSpanStartAttributes<"pi.session.write">;
|
||||
type WriteEnd = HarnessSpanEndAttributes<"pi.session.write">;
|
||||
expectTypeOf<RunStart["pi.operation.kind"]>().toEqualTypeOf<"run">();
|
||||
expectTypeOf<RunEnd["pi.operation.outcome"]>().toEqualTypeOf<
|
||||
"completed" | "aborted" | "failed" | "suspended" | undefined
|
||||
>();
|
||||
const writeStart = {
|
||||
"pi.session.id": "session",
|
||||
"pi.session.item_count": 2,
|
||||
"pi.session.item_kinds": ["entry", "value", "list"],
|
||||
} satisfies WriteStart;
|
||||
const writeEnd = {
|
||||
"pi.session.first_seq": 1,
|
||||
"pi.session.last_seq": 2,
|
||||
} satisfies WriteEnd;
|
||||
expectTypeOf(writeStart["pi.session.item_count"]).toEqualTypeOf<number>();
|
||||
expectTypeOf(writeEnd["pi.session.last_seq"]).toEqualTypeOf<number>();
|
||||
|
||||
const telemetryContext: TelemetryContext = NOOP_TELEMETRY_CONTEXT;
|
||||
const context = withTelemetryContext(telemetryContext, BACKGROUND_CONTEXT);
|
||||
await startHarnessSpan(
|
||||
"pi.harness.run",
|
||||
{
|
||||
"pi.session.id": "session",
|
||||
"pi.lane.name": "main",
|
||||
"pi.operation.id": "operation",
|
||||
"pi.operation.kind": "run",
|
||||
"pi.operation.recovery": false,
|
||||
},
|
||||
(span) => {
|
||||
span.setAttributes({ "pi.operation.outcome": "completed" });
|
||||
span.setAttributes({});
|
||||
// @ts-expect-error the harness schema declares no span events
|
||||
span.addEvent("result");
|
||||
},
|
||||
context,
|
||||
);
|
||||
|
||||
const compileTimeFailures = () => {
|
||||
const extraRunAttributes = {
|
||||
"pi.session.id": "session",
|
||||
"pi.lane.name": "main",
|
||||
"pi.operation.id": "operation",
|
||||
"pi.operation.kind": "run",
|
||||
"pi.operation.recovery": false,
|
||||
"pi.unknown": true,
|
||||
} as const;
|
||||
// @ts-expect-error variables with unknown attributes are rejected
|
||||
void startHarnessSpan("pi.harness.run", extraRunAttributes, () => {}, context);
|
||||
void startHarnessSpan(
|
||||
"pi.harness.checkpoint",
|
||||
{
|
||||
"pi.lane.name": "main",
|
||||
"pi.operation.id": "operation",
|
||||
"pi.checkpoint.kind": "normal",
|
||||
},
|
||||
(span) => {
|
||||
// @ts-expect-error empty end schemas reject every attribute
|
||||
span.setAttributes({ "pi.unknown": true });
|
||||
},
|
||||
context,
|
||||
);
|
||||
void startHarnessSpan(
|
||||
"pi.harness.run",
|
||||
{
|
||||
"pi.session.id": "session",
|
||||
"pi.lane.name": "main",
|
||||
"pi.operation.id": "operation",
|
||||
// @ts-expect-error run spans accept only the run operation kind
|
||||
"pi.operation.kind": "navigation",
|
||||
"pi.operation.recovery": false,
|
||||
},
|
||||
() => {},
|
||||
context,
|
||||
);
|
||||
// @ts-expect-error missing required run start attributes
|
||||
void startHarnessSpan("pi.harness.run", {}, () => {}, context);
|
||||
};
|
||||
expectTypeOf(compileTimeFailures).toBeFunction();
|
||||
});
|
||||
});
|
||||
782
third_party/pi/packages/agent/test/harness/tools.test.ts
vendored
Normal file
782
third_party/pi/packages/agent/test/harness/tools.test.ts
vendored
Normal file
@@ -0,0 +1,782 @@
|
||||
import { symlink } from "node:fs/promises";
|
||||
import { applyPatch } from "diff";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import { BACKGROUND_CONTEXT, type Context, withAbortSignal } from "../../src/harness/context.ts";
|
||||
import { NodeExecutionEnv } from "../../src/harness/env/nodejs.ts";
|
||||
import { type BashToolDetails, createBashTool } from "../../src/harness/tools/bash.ts";
|
||||
import { createEditTool } from "../../src/harness/tools/edit.ts";
|
||||
import { createReadTool } from "../../src/harness/tools/read.ts";
|
||||
import { createWriteTool } from "../../src/harness/tools/write.ts";
|
||||
import {
|
||||
ExecutionError,
|
||||
err,
|
||||
type FileError,
|
||||
getOrThrow,
|
||||
ok,
|
||||
type Result,
|
||||
type ShellExecOptions,
|
||||
type ShellExecResult,
|
||||
} from "../../src/harness/types.ts";
|
||||
import { DEFAULT_MAX_BYTES, DEFAULT_MAX_LINES, truncateTail } from "../../src/harness/utils/truncate.ts";
|
||||
import { createTempDir } from "./session-test-utils.ts";
|
||||
|
||||
const noUpdate = () => {};
|
||||
|
||||
function textOutput(result: { content: Array<{ type: string; text?: string }> }): string {
|
||||
return result.content.flatMap((part) => (part.type === "text" ? [part.text ?? ""] : [])).join("\n");
|
||||
}
|
||||
|
||||
function createContext() {
|
||||
const env = new NodeExecutionEnv({ cwd: createTempDir() });
|
||||
return { env };
|
||||
}
|
||||
|
||||
function deferred(): { promise: Promise<void>; resolve: () => void } {
|
||||
let resolve = () => {};
|
||||
const promise = new Promise<void>((resolvePromise) => {
|
||||
resolve = resolvePromise;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
function delay(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
class SlowReadExecutionEnv extends NodeExecutionEnv {
|
||||
override async readTextFile(path: string, context: Context): Promise<Result<string, FileError>> {
|
||||
await delay(20);
|
||||
return super.readTextFile(path, context);
|
||||
}
|
||||
}
|
||||
|
||||
class BlockingWriteExecutionEnv extends NodeExecutionEnv {
|
||||
readonly firstWriteStarted = deferred();
|
||||
readonly finishFirstWrite = deferred();
|
||||
secondWriteStarted = false;
|
||||
|
||||
override async writeFile(
|
||||
path: string,
|
||||
content: string | Uint8Array,
|
||||
context: Context,
|
||||
): Promise<Result<void, FileError>> {
|
||||
if (content === "first\n") {
|
||||
this.firstWriteStarted.resolve();
|
||||
await this.finishFirstWrite.promise;
|
||||
} else if (content === "second\n") {
|
||||
this.secondWriteStarted = true;
|
||||
}
|
||||
return super.writeFile(path, content, context);
|
||||
}
|
||||
}
|
||||
|
||||
class BlockingEditExecutionEnv extends NodeExecutionEnv {
|
||||
readonly firstEditWriteStarted = deferred();
|
||||
readonly finishFirstEditWrite = deferred();
|
||||
firstEditWriteSettled = false;
|
||||
secondEditWriteStarted = false;
|
||||
|
||||
override async writeFile(
|
||||
path: string,
|
||||
content: string | Uint8Array,
|
||||
context: Context,
|
||||
): Promise<Result<void, FileError>> {
|
||||
if (content === "ALPHA\nbeta\n") {
|
||||
this.firstEditWriteStarted.resolve();
|
||||
await this.finishFirstEditWrite.promise;
|
||||
const result = await super.writeFile(path, content, BACKGROUND_CONTEXT);
|
||||
this.firstEditWriteSettled = true;
|
||||
return result;
|
||||
}
|
||||
if (content === "ALPHA\nBETA\n" || content === "alpha\nBETA\n") {
|
||||
this.secondEditWriteStarted = true;
|
||||
}
|
||||
return super.writeFile(path, content, context);
|
||||
}
|
||||
}
|
||||
|
||||
function fakeShellOutput(text: string, options: ShellExecOptions | undefined, spillPath?: string): ShellExecResult {
|
||||
const limits = options?.capture?.limits ?? {
|
||||
maxBytes: DEFAULT_MAX_BYTES,
|
||||
maxLines: DEFAULT_MAX_LINES,
|
||||
retain: "tail" as const,
|
||||
};
|
||||
const truncated = truncateTail(text, limits);
|
||||
const { content, ...truncation } = truncated;
|
||||
options?.onUpdate?.(
|
||||
{
|
||||
kind: "replace",
|
||||
output: { text: content, truncation, ...(spillPath === undefined ? {} : { spillPath }) },
|
||||
},
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
return { exitCode: 0, truncation, ...(spillPath === undefined ? {} : { spillPath }) };
|
||||
}
|
||||
|
||||
class LateOutputExecutionEnv extends NodeExecutionEnv {
|
||||
override async exec(
|
||||
_command: string,
|
||||
options: ShellExecOptions | undefined,
|
||||
context: Context,
|
||||
): Promise<Result<ShellExecResult, ExecutionError>> {
|
||||
const result = fakeShellOutput("before\n", options);
|
||||
setTimeout(() => {
|
||||
const truncation = truncateTail("before\nlate\n", options?.capture?.limits);
|
||||
options?.onUpdate?.({ kind: "replace", output: { text: truncation.content, truncation } }, context);
|
||||
}, 0);
|
||||
return ok(result);
|
||||
}
|
||||
}
|
||||
|
||||
const TRUNCATED_OUTPUT_LINES = DEFAULT_MAX_LINES + 1;
|
||||
const invocation = {
|
||||
invocationId: "test-result",
|
||||
operationId: "test-operation",
|
||||
turnId: "test-turn",
|
||||
getMemo: async () => undefined,
|
||||
setMemo: async () => {},
|
||||
};
|
||||
|
||||
class CheckpointOutputExecutionEnv extends NodeExecutionEnv {
|
||||
override async exec(
|
||||
_command: string,
|
||||
options: ShellExecOptions | undefined,
|
||||
_context: Context,
|
||||
): Promise<Result<ShellExecResult, ExecutionError>> {
|
||||
fakeShellOutput("one\n", options);
|
||||
await delay(2_100);
|
||||
fakeShellOutput("one\ntwo\n", options);
|
||||
await delay(100);
|
||||
fakeShellOutput("one\ntwo\nthree\n", options);
|
||||
await delay(2_000);
|
||||
return ok(fakeShellOutput("one\ntwo\nthree\nfour\n", options));
|
||||
}
|
||||
}
|
||||
|
||||
class TimeoutOutputExecutionEnv extends NodeExecutionEnv {
|
||||
override async exec(
|
||||
_command: string,
|
||||
options: ShellExecOptions | undefined,
|
||||
context: Context,
|
||||
): Promise<Result<ShellExecResult, ExecutionError>> {
|
||||
const output = `${Array.from({ length: TRUNCATED_OUTPUT_LINES }, (_, index) => `line-${index + 1}`).join("\n")}\n`;
|
||||
const spillPath = getOrThrow(await this.createTempFile({ prefix: "timeout-", suffix: ".log" }, context));
|
||||
getOrThrow(await this.writeFile(spillPath, output, context));
|
||||
fakeShellOutput(output, options, spillPath);
|
||||
return err(new ExecutionError("timeout", `timeout:${options?.timeout}`));
|
||||
}
|
||||
}
|
||||
|
||||
function createTinyBmp(): Uint8Array {
|
||||
const bytes = new Uint8Array(58);
|
||||
const view = new DataView(bytes.buffer);
|
||||
bytes[0] = 0x42;
|
||||
bytes[1] = 0x4d;
|
||||
view.setUint32(2, bytes.length, true);
|
||||
view.setUint32(10, 54, true);
|
||||
view.setUint32(14, 40, true);
|
||||
view.setInt32(18, 1, true);
|
||||
view.setInt32(22, 1, true);
|
||||
view.setUint16(26, 1, true);
|
||||
view.setUint16(28, 24, true);
|
||||
view.setUint32(34, 4, true);
|
||||
return bytes;
|
||||
}
|
||||
|
||||
describe("AgentHarness tools", () => {
|
||||
describe("read", () => {
|
||||
it("reads text with offsets, limits, and continuation notices", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(
|
||||
await context.env.writeFile(
|
||||
"test.txt",
|
||||
Array.from({ length: 100 }, (_, index) => `Line ${index + 1}`).join("\n"),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
|
||||
const result = await createReadTool().execute(
|
||||
"read-1",
|
||||
{ path: "test.txt", offset: 41, limit: 20 },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
const output = textOutput(result);
|
||||
|
||||
expect(output).not.toContain("Line 40");
|
||||
expect(output).toContain("Line 41");
|
||||
expect(output).toContain("Line 60");
|
||||
expect(output).not.toContain("Line 61");
|
||||
expect(output).toContain("[40 more lines in file. Use offset=61 to continue.]");
|
||||
});
|
||||
|
||||
it("truncates large text by line count", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(
|
||||
await context.env.writeFile(
|
||||
"large.txt",
|
||||
Array.from({ length: 2500 }, (_, index) => `Line ${index + 1}`).join("\n"),
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
|
||||
const result = await createReadTool().execute(
|
||||
"read-2",
|
||||
{ path: "large.txt" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toContain("[Showing lines 1-2000 of 2500. Use offset=2001 to continue.]");
|
||||
expect(result.details?.truncation).toMatchObject({
|
||||
truncated: true,
|
||||
truncatedBy: "lines",
|
||||
totalLines: 2500,
|
||||
outputLines: 2000,
|
||||
});
|
||||
});
|
||||
|
||||
it("does not count a trailing newline as an extra line at the truncation limit", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(
|
||||
await context.env.writeFile(
|
||||
"exact.txt",
|
||||
`${Array.from({ length: 2000 }, () => "x").join("\n")}\n`,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
);
|
||||
|
||||
const result = await createReadTool().execute(
|
||||
"read-exact",
|
||||
{ path: "exact.txt" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(result.details).toBeUndefined();
|
||||
expect(textOutput(result)).not.toContain("Use offset=");
|
||||
});
|
||||
|
||||
it("rejects offsets beyond the file", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(await context.env.writeFile("short.txt", "one\ntwo\nthree", BACKGROUND_CONTEXT));
|
||||
|
||||
await expect(
|
||||
createReadTool().execute(
|
||||
"read-3",
|
||||
{ path: "short.txt", offset: 100 },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow("Offset 100 is beyond end of file (3 lines total)");
|
||||
});
|
||||
|
||||
it("detects supported images by content", async () => {
|
||||
const context = createContext();
|
||||
const png = Uint8Array.from(
|
||||
Buffer.from(
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR4nGNgYGD4DwABBAEAX+XDSwAAAABJRU5ErkJggg==",
|
||||
"base64",
|
||||
),
|
||||
);
|
||||
getOrThrow(await context.env.writeFile("image.txt", png, BACKGROUND_CONTEXT));
|
||||
|
||||
const result = await createReadTool().execute(
|
||||
"read-4",
|
||||
{ path: "image.txt" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toContain("Read image file [image/png]");
|
||||
expect(result.content).toContainEqual({
|
||||
type: "image",
|
||||
data: Buffer.from(png).toString("base64"),
|
||||
mimeType: "image/png",
|
||||
});
|
||||
});
|
||||
|
||||
it("delegates image conversion and resizing to an injected processor", async () => {
|
||||
const context = createContext();
|
||||
const bmp = createTinyBmp();
|
||||
getOrThrow(await context.env.writeFile("image.bmp", bmp, BACKGROUND_CONTEXT));
|
||||
let received: { bytes: Uint8Array; mimeType: string; autoResizeImages: boolean } | undefined;
|
||||
const tool = createReadTool({
|
||||
autoResizeImages: false,
|
||||
imageProcessor: async (bytes, mimeType, options) => {
|
||||
received = { bytes, mimeType, autoResizeImages: options.autoResizeImages };
|
||||
return {
|
||||
ok: true,
|
||||
data: "converted",
|
||||
mimeType: "image/png",
|
||||
hints: ["[Image converted from image/bmp to image/png.]"],
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const result = await tool.execute(
|
||||
"read-bmp",
|
||||
{ path: "image.bmp" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(received).toMatchObject({ mimeType: "image/bmp", autoResizeImages: false });
|
||||
expect(Array.from(received?.bytes ?? [])).toEqual(Array.from(bmp));
|
||||
expect(textOutput(result)).toContain("[Image converted from image/bmp to image/png.]");
|
||||
expect(result.content).toContainEqual({ type: "image", data: "converted", mimeType: "image/png" });
|
||||
});
|
||||
});
|
||||
|
||||
describe("write", () => {
|
||||
it("writes files and creates parent directories", async () => {
|
||||
const context = createContext();
|
||||
const result = await createWriteTool().execute(
|
||||
"write-1",
|
||||
{ path: "nested/dir/file.txt", content: "hello" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toBe("Successfully wrote to nested/dir/file.txt");
|
||||
expect(getOrThrow(await context.env.readTextFile("nested/dir/file.txt", BACKGROUND_CONTEXT))).toBe("hello");
|
||||
});
|
||||
|
||||
it("keeps the mutation queue locked until an aborted write settles", async () => {
|
||||
const env = new BlockingWriteExecutionEnv({ cwd: createTempDir() });
|
||||
const tool = createWriteTool();
|
||||
const controller = new AbortController();
|
||||
const firstWrite = tool.execute(
|
||||
"write-first",
|
||||
{ path: "file.txt", content: "first\n" },
|
||||
noUpdate,
|
||||
{
|
||||
env,
|
||||
},
|
||||
invocation,
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
);
|
||||
await env.firstWriteStarted.promise;
|
||||
controller.abort();
|
||||
const secondWrite = tool.execute(
|
||||
"write-second",
|
||||
{ path: "file.txt", content: "second\n" },
|
||||
noUpdate,
|
||||
{ env },
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
await delay(20);
|
||||
expect(env.secondWriteStarted).toBe(false);
|
||||
env.finishFirstWrite.resolve();
|
||||
await expect(firstWrite).rejects.toThrow();
|
||||
await secondWrite;
|
||||
expect(getOrThrow(await env.readTextFile("file.txt", BACKGROUND_CONTEXT))).toBe("second\n");
|
||||
});
|
||||
});
|
||||
|
||||
describe("edit", () => {
|
||||
it("applies disjoint edits and returns both diff formats", async () => {
|
||||
const context = createContext();
|
||||
const original = "alpha\nbeta\ngamma\ndelta\n";
|
||||
getOrThrow(await context.env.writeFile("edit.txt", original, BACKGROUND_CONTEXT));
|
||||
|
||||
const result = await createEditTool().execute(
|
||||
"edit-1",
|
||||
{
|
||||
path: "edit.txt",
|
||||
edits: [
|
||||
{ oldText: "alpha\n", newText: "ALPHA\n" },
|
||||
{ oldText: "gamma\n", newText: "GAMMA\n" },
|
||||
],
|
||||
},
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toBe("Successfully replaced 2 block(s) in edit.txt.");
|
||||
expect(result.details?.diff).toContain("ALPHA");
|
||||
expect(result.details?.diff).toContain("GAMMA");
|
||||
expect(applyPatch(original, result.details?.patch ?? "")).toBe("ALPHA\nbeta\nGAMMA\ndelta\n");
|
||||
expect(getOrThrow(await context.env.readTextFile("edit.txt", BACKGROUND_CONTEXT))).toBe(
|
||||
"ALPHA\nbeta\nGAMMA\ndelta\n",
|
||||
);
|
||||
});
|
||||
|
||||
it("matches all edits against the original and rejects overlaps", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(await context.env.writeFile("edit.txt", "one\ntwo\nthree\n", BACKGROUND_CONTEXT));
|
||||
|
||||
await expect(
|
||||
createEditTool().execute(
|
||||
"edit-2",
|
||||
{
|
||||
path: "edit.txt",
|
||||
edits: [
|
||||
{ oldText: "one\ntwo\n", newText: "ONE\nTWO\n" },
|
||||
{ oldText: "two\nthree\n", newText: "TWO\nTHREE\n" },
|
||||
],
|
||||
},
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow(/overlap/);
|
||||
expect(getOrThrow(await context.env.readTextFile("edit.txt", BACKGROUND_CONTEXT))).toBe("one\ntwo\nthree\n");
|
||||
});
|
||||
|
||||
it("rejects missing and duplicate target text", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(await context.env.writeFile("edit.txt", "foo foo foo", BACKGROUND_CONTEXT));
|
||||
const tool = createEditTool();
|
||||
|
||||
await expect(
|
||||
tool.execute(
|
||||
"edit-3",
|
||||
{ path: "edit.txt", edits: [{ oldText: "bar", newText: "baz" }] },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow(/Could not find the exact text/);
|
||||
await expect(
|
||||
tool.execute(
|
||||
"edit-4",
|
||||
{ path: "edit.txt", edits: [{ oldText: "foo", newText: "bar" }] },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow(/Found 3 occurrences/);
|
||||
});
|
||||
|
||||
it("keeps the mutation queue locked until an aborted edit write settles", async () => {
|
||||
const env = new BlockingEditExecutionEnv({ cwd: createTempDir() });
|
||||
getOrThrow(await env.writeFile("file.txt", "alpha\nbeta\n", BACKGROUND_CONTEXT));
|
||||
const tool = createEditTool();
|
||||
const controller = new AbortController();
|
||||
const firstEdit = tool.execute(
|
||||
"edit-first",
|
||||
{ path: "file.txt", edits: [{ oldText: "alpha", newText: "ALPHA" }] },
|
||||
noUpdate,
|
||||
{ env },
|
||||
invocation,
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
);
|
||||
await env.firstEditWriteStarted.promise;
|
||||
controller.abort();
|
||||
const secondEdit = tool.execute(
|
||||
"edit-second",
|
||||
{ path: "file.txt", edits: [{ oldText: "beta", newText: "BETA" }] },
|
||||
noUpdate,
|
||||
{ env },
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
await delay(20);
|
||||
expect(env.secondEditWriteStarted).toBe(false);
|
||||
env.finishFirstEditWrite.resolve();
|
||||
await expect(firstEdit).rejects.toThrow("Operation aborted");
|
||||
await secondEdit;
|
||||
expect(env.firstEditWriteSettled).toBe(true);
|
||||
expect(getOrThrow(await env.readTextFile("file.txt", BACKGROUND_CONTEXT))).toBe("ALPHA\nBETA\n");
|
||||
});
|
||||
|
||||
it("serializes concurrent edits through canonical and symlink paths", async () => {
|
||||
const env = new SlowReadExecutionEnv({ cwd: createTempDir() });
|
||||
getOrThrow(await env.writeFile("target.txt", "alpha\nbeta\ngamma\n", BACKGROUND_CONTEXT));
|
||||
await symlink("target.txt", `${env.cwd}/link.txt`);
|
||||
const tool = createEditTool();
|
||||
|
||||
await Promise.all([
|
||||
tool.execute(
|
||||
"edit-target",
|
||||
{ path: "target.txt", edits: [{ oldText: "alpha", newText: "ALPHA" }] },
|
||||
noUpdate,
|
||||
{ env },
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
tool.execute(
|
||||
"edit-link",
|
||||
{ path: "link.txt", edits: [{ oldText: "beta", newText: "BETA" }] },
|
||||
noUpdate,
|
||||
{ env },
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
]);
|
||||
|
||||
expect(getOrThrow(await env.readTextFile("target.txt", BACKGROUND_CONTEXT))).toBe("ALPHA\nBETA\ngamma\n");
|
||||
});
|
||||
|
||||
it("edits regular files through symlinks", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(await context.env.writeFile("target.txt", "before\n", BACKGROUND_CONTEXT));
|
||||
await symlink("target.txt", `${context.env.cwd}/link.txt`);
|
||||
|
||||
await createEditTool().execute(
|
||||
"edit-symlink",
|
||||
{ path: "link.txt", edits: [{ oldText: "before", newText: "after" }] },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(getOrThrow(await context.env.readTextFile("target.txt", BACKGROUND_CONTEXT))).toBe("after\n");
|
||||
});
|
||||
|
||||
it("preserves BOM and CRLF line endings", async () => {
|
||||
const context = createContext();
|
||||
getOrThrow(await context.env.writeFile("edit.txt", "\uFEFFone\r\ntwo\r\n", BACKGROUND_CONTEXT));
|
||||
|
||||
await createEditTool().execute(
|
||||
"edit-5",
|
||||
{ path: "edit.txt", edits: [{ oldText: "two", newText: "TWO" }] },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(getOrThrow(await context.env.readTextFile("edit.txt", BACKGROUND_CONTEXT))).toBe(
|
||||
"\uFEFFone\r\nTWO\r\n",
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("bash", () => {
|
||||
it("executes commands and combines stdout and stderr", async () => {
|
||||
const context = createContext();
|
||||
const result = await createBashTool().execute(
|
||||
"bash-1",
|
||||
{ command: "printf out; printf err >&2" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toContain("out");
|
||||
expect(textOutput(result)).toContain("err");
|
||||
});
|
||||
|
||||
it("reports nonzero exits and timeouts", async () => {
|
||||
const context = createContext();
|
||||
const tool = createBashTool();
|
||||
|
||||
await expect(
|
||||
tool.execute(
|
||||
"bash-2",
|
||||
{ command: "printf failed; exit 7" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow(/failed[\s\S]*Command exited with code 7/);
|
||||
await expect(
|
||||
tool.execute(
|
||||
"bash-3",
|
||||
{ command: "sleep 2", timeout: 0.01 },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
),
|
||||
).rejects.toThrow(/Command timed out after 0.01 seconds/);
|
||||
});
|
||||
|
||||
it("preserves truncated output when a command times out", async () => {
|
||||
const context = { env: new TimeoutOutputExecutionEnv({ cwd: createTempDir() }) };
|
||||
let error: unknown;
|
||||
try {
|
||||
await createBashTool().execute(
|
||||
"bash-timeout-output",
|
||||
{ command: "emit-output-then-time-out", timeout: 0.05 },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
} catch (cause) {
|
||||
error = cause;
|
||||
}
|
||||
|
||||
expect(error).toBeInstanceOf(Error);
|
||||
const message = (error as Error).message;
|
||||
expect(message).toContain("Command timed out after 0.05 seconds");
|
||||
const fullOutputPath = message.match(/Full output: ([^\]\n]+)/)?.[1];
|
||||
expect(fullOutputPath).toBeDefined();
|
||||
const fullOutput = getOrThrow(await context.env.readTextFile(fullOutputPath!, BACKGROUND_CONTEXT));
|
||||
expect(fullOutput).toContain("line-1\nline-2");
|
||||
expect(fullOutput).toContain(`line-${DEFAULT_MAX_LINES}\nline-${TRUNCATED_OUTPUT_LINES}`);
|
||||
});
|
||||
|
||||
it("ignores output callbacks after execution settles", async () => {
|
||||
const env = new LateOutputExecutionEnv({ cwd: createTempDir() });
|
||||
const updates: string[] = [];
|
||||
const result = await createBashTool().execute(
|
||||
"bash-late",
|
||||
{ command: "late" },
|
||||
(update) => updates.push(textOutput(update)),
|
||||
{ env },
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await new Promise((resolve) => setTimeout(resolve, 20));
|
||||
|
||||
expect(textOutput(result)).toBe("before\n");
|
||||
expect(updates.some((update) => update.includes("late"))).toBe(false);
|
||||
});
|
||||
|
||||
it("reports the total size of an oversized final line", async () => {
|
||||
const context = createContext();
|
||||
const result = await createBashTool().execute(
|
||||
"bash-long-line",
|
||||
{ command: "printf '%060000d' 0" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toMatch(/Showing last 50\.0KB of line 1 \(line is 58\.6KB\)\. Full output:/);
|
||||
});
|
||||
|
||||
it("prepares command, cwd, and an explicit environment with the turn context", async () => {
|
||||
const env = new NodeExecutionEnv({
|
||||
cwd: createTempDir(),
|
||||
shellEnv: { PI_BASH_PREPARE_INHERITED: "inherited" },
|
||||
});
|
||||
getOrThrow(await env.createDir("workspace", undefined, BACKGROUND_CONTEXT));
|
||||
const context = { env, workspace: `${env.cwd}/workspace` };
|
||||
const controller = new AbortController();
|
||||
let receivedContext: typeof context | undefined;
|
||||
let receivedSignal: AbortSignal | undefined;
|
||||
const tool = createBashTool<typeof context>({
|
||||
commandPrefix: "prefix=ready",
|
||||
prepare: async (execution, turnContext, callContext) => {
|
||||
receivedContext = turnContext;
|
||||
receivedSignal = callContext.abortSignal;
|
||||
execution.cwd = turnContext.workspace;
|
||||
execution.env = { PI_BASH_PREPARE_EXPLICIT: "explicit" };
|
||||
execution.inheritEnv = false;
|
||||
execution.command += `\nprintf '%s:%s:%s:%s' "$prefix" "\${PI_BASH_PREPARE_INHERITED-}" "$PI_BASH_PREPARE_EXPLICIT" "$PWD"`;
|
||||
},
|
||||
});
|
||||
|
||||
const result = await tool.execute(
|
||||
"bash-prepare",
|
||||
{ command: ":" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
withAbortSignal(controller.signal, BACKGROUND_CONTEXT),
|
||||
);
|
||||
|
||||
expect(receivedContext).toBe(context);
|
||||
expect(receivedSignal).toBe(controller.signal);
|
||||
expect(textOutput(result)).toBe(
|
||||
`ready::explicit:${getOrThrow(await env.canonicalPath(context.workspace, BACKGROUND_CONTEXT))}`,
|
||||
);
|
||||
});
|
||||
|
||||
it("supports command prefixes", async () => {
|
||||
const context = createContext();
|
||||
const result = await createBashTool({ commandPrefix: "value=hello" }).execute(
|
||||
"bash-4",
|
||||
{ command: "printf $value" },
|
||||
noUpdate,
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(textOutput(result)).toBe("hello");
|
||||
});
|
||||
|
||||
it("requests distinct bounded checkpoints at most every two seconds", async () => {
|
||||
vi.useFakeTimers({ now: 0 });
|
||||
try {
|
||||
const env = new CheckpointOutputExecutionEnv({ cwd: createTempDir() });
|
||||
const checkpoints: string[] = [];
|
||||
const execution = createBashTool().execute(
|
||||
"bash-checkpoints",
|
||||
{ command: "controlled" },
|
||||
(update, options) => {
|
||||
if (options?.checkpoint === true) checkpoints.push(textOutput(update));
|
||||
},
|
||||
{ env },
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
await vi.advanceTimersByTimeAsync(4_500);
|
||||
await execution;
|
||||
|
||||
expect(checkpoints).toEqual(["one\ntwo\n", "one\ntwo\nthree\nfour\n"]);
|
||||
} finally {
|
||||
vi.useRealTimers();
|
||||
}
|
||||
});
|
||||
|
||||
it("coalesces updates and persists truncated full output", async () => {
|
||||
const context = createContext();
|
||||
const updates: Array<{
|
||||
content: Array<{ type: "text"; text: string } | { type: "image"; data: string; mimeType: string }>;
|
||||
details?: BashToolDetails;
|
||||
}> = [];
|
||||
const result = await createBashTool().execute(
|
||||
"bash-5",
|
||||
{ command: "i=1; while [ $i -le 3000 ]; do echo line-$i; i=$((i + 1)); done" },
|
||||
(update) => updates.push(update),
|
||||
context,
|
||||
invocation,
|
||||
BACKGROUND_CONTEXT,
|
||||
);
|
||||
|
||||
expect(updates.length).toBeLessThan(25);
|
||||
expect(result.details?.truncation).toMatchObject({
|
||||
truncated: true,
|
||||
truncatedBy: "lines",
|
||||
totalLines: 3000,
|
||||
outputLines: 2000,
|
||||
});
|
||||
expect(textOutput(result)).toContain("line-3000");
|
||||
expect(result.details?.fullOutputPath).toBeDefined();
|
||||
const finalUpdate = updates.at(-1);
|
||||
expect(finalUpdate ? textOutput(finalUpdate) : "").toContain("line-3000");
|
||||
expect(finalUpdate?.details).toMatchObject({
|
||||
truncation: { totalLines: 3000, totalBytes: expect.any(Number) },
|
||||
fullOutputPath: result.details?.fullOutputPath,
|
||||
});
|
||||
const fullOutput = getOrThrow(
|
||||
await context.env.readTextFile(result.details!.fullOutputPath!, BACKGROUND_CONTEXT),
|
||||
);
|
||||
expect(fullOutput).toContain("line-1\nline-2");
|
||||
expect(fullOutput).toContain("line-2999\nline-3000");
|
||||
});
|
||||
});
|
||||
});
|
||||
178
third_party/pi/packages/agent/test/harness/truncate.test.ts
vendored
Normal file
178
third_party/pi/packages/agent/test/harness/truncate.test.ts
vendored
Normal file
@@ -0,0 +1,178 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { truncateHead, truncateTail } from "../../src/harness/utils/truncate.ts";
|
||||
|
||||
const encoder = new TextEncoder();
|
||||
|
||||
function byteLength(content: string): number {
|
||||
return encoder.encode(content).length;
|
||||
}
|
||||
|
||||
function bufferTail(content: string, maxBytes: number): string {
|
||||
const bytes = Buffer.from(content, "utf8");
|
||||
if (bytes.length <= maxBytes) return content;
|
||||
let start = bytes.length - maxBytes;
|
||||
while (start < bytes.length && (bytes[start] & 0xc0) === 0x80) start++;
|
||||
return bytes.subarray(start).toString("utf8");
|
||||
}
|
||||
|
||||
function assertMatchesBufferTail(input: string, maxByteValues?: readonly number[]): void {
|
||||
const totalBytes = Buffer.byteLength(input, "utf8");
|
||||
const values = maxByteValues ?? Array.from({ length: totalBytes + 5 }, (_, maxBytes) => maxBytes);
|
||||
for (const maxBytes of values) {
|
||||
const result = truncateTail(input, { maxBytes, maxLines: 10 });
|
||||
const expected = bufferTail(input, maxBytes);
|
||||
if (result.content !== expected) {
|
||||
throw new Error(
|
||||
`tail mismatch input=${JSON.stringify(input)} maxBytes=${maxBytes} expected=${JSON.stringify(expected)} actual=${JSON.stringify(result.content)}`,
|
||||
);
|
||||
}
|
||||
const outputBytes = Buffer.byteLength(result.content, "utf8");
|
||||
if (outputBytes > maxBytes) {
|
||||
throw new Error(
|
||||
`tail output exceeded byte limit input=${JSON.stringify(input)} maxBytes=${maxBytes} outputBytes=${outputBytes}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function sampledByteLimits(input: string): number[] {
|
||||
const totalBytes = Buffer.byteLength(input, "utf8");
|
||||
const candidates = [
|
||||
0,
|
||||
1,
|
||||
2,
|
||||
3,
|
||||
4,
|
||||
5,
|
||||
8,
|
||||
Math.floor(totalBytes / 2) - 1,
|
||||
Math.floor(totalBytes / 2),
|
||||
Math.floor(totalBytes / 2) + 1,
|
||||
totalBytes - 8,
|
||||
totalBytes - 5,
|
||||
totalBytes - 4,
|
||||
totalBytes - 3,
|
||||
totalBytes - 2,
|
||||
totalBytes - 1,
|
||||
totalBytes,
|
||||
totalBytes + 1,
|
||||
totalBytes + 4,
|
||||
];
|
||||
return [...new Set(candidates.filter((value) => value >= 0))].sort((a, b) => a - b);
|
||||
}
|
||||
|
||||
describe("truncate utilities", () => {
|
||||
it("counts UTF-8 bytes without Node Buffer", () => {
|
||||
const content = "aé🙂\nb";
|
||||
const result = truncateHead(content, { maxBytes: 100, maxLines: 10 });
|
||||
|
||||
expect(result.truncated).toBe(false);
|
||||
expect(result.totalBytes).toBe(byteLength(content));
|
||||
expect(result.outputBytes).toBe(byteLength(content));
|
||||
expect(result.totalBytes).toBe(9);
|
||||
});
|
||||
|
||||
it("does not count a trailing newline as an extra line", () => {
|
||||
const content = `${Array.from({ length: 3 }, () => "line").join("\n")}\n`;
|
||||
const head = truncateHead(content, { maxBytes: 100, maxLines: 3 });
|
||||
const tail = truncateTail(content, { maxBytes: 100, maxLines: 3 });
|
||||
|
||||
expect(head).toMatchObject({ truncated: false, totalLines: 3, outputLines: 3 });
|
||||
expect(tail).toMatchObject({ truncated: false, totalLines: 3, outputLines: 3 });
|
||||
});
|
||||
|
||||
it("truncates head on UTF-8 byte limits without partial lines", () => {
|
||||
const content = "éé\nabc";
|
||||
const result = truncateHead(content, { maxBytes: 4, maxLines: 10 });
|
||||
|
||||
expect(result.content).toBe("éé");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.outputBytes).toBe(4);
|
||||
expect(result.firstLineExceedsLimit).toBe(false);
|
||||
});
|
||||
|
||||
it("reports head truncation when the first line exceeds the byte limit", () => {
|
||||
const result = truncateHead("éé\nabc", { maxBytes: 3, maxLines: 10 });
|
||||
|
||||
expect(result.content).toBe("");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.firstLineExceedsLimit).toBe(true);
|
||||
});
|
||||
|
||||
it("truncates tail on UTF-8 boundaries when only a partial last line fits", () => {
|
||||
const result = truncateTail("aé🙂b", { maxBytes: 5, maxLines: 10 });
|
||||
|
||||
expect(result.content).toBe("🙂b");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.lastLinePartial).toBe(true);
|
||||
expect(result.outputBytes).toBe(5);
|
||||
});
|
||||
|
||||
it("truncates an oversized single line with a trailing newline", () => {
|
||||
const input = `${"X".repeat(300_000)}\n`;
|
||||
const result = truncateTail(input, { maxBytes: 1024, maxLines: 100 });
|
||||
|
||||
expect(result.content).toBe("X".repeat(1024));
|
||||
expect(result.outputBytes).toBe(1024);
|
||||
expect(result.outputLines).toBe(1);
|
||||
expect(result.lastLinePartial).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
});
|
||||
|
||||
it("drops an oversized trailing character when it cannot fit in tail byte limit", () => {
|
||||
const result = truncateTail("abc🙂", { maxBytes: 3, maxLines: 10 });
|
||||
|
||||
expect(result.content).toBe("");
|
||||
expect(result.truncated).toBe(true);
|
||||
expect(result.truncatedBy).toBe("bytes");
|
||||
expect(result.lastLinePartial).toBe(true);
|
||||
expect(result.outputBytes).toBe(0);
|
||||
});
|
||||
|
||||
it("matches Buffer tail truncation semantics for surrogate edge cases", () => {
|
||||
const inputs = ["a\ud83d", "\ude42b", "a\ude42b", "\ud83d\ud83d\ude42", "\ud83d\ude42\ude42", "👩💻"];
|
||||
for (const input of inputs) assertMatchesBufferTail(input);
|
||||
});
|
||||
|
||||
it("matches Buffer tail truncation semantics across deterministic fuzz cases", () => {
|
||||
const alphabet = [
|
||||
"a",
|
||||
"\u007f",
|
||||
"\u0080",
|
||||
"é",
|
||||
"\u07ff",
|
||||
"\u0800",
|
||||
"中",
|
||||
"\ud7ff",
|
||||
"\ud800",
|
||||
"\ud83d",
|
||||
"\udc00",
|
||||
"\ude42",
|
||||
"🙂",
|
||||
"\ue000",
|
||||
"\uffff",
|
||||
];
|
||||
|
||||
function checkExhaustive(prefix: string, depth: number): void {
|
||||
assertMatchesBufferTail(prefix, sampledByteLimits(prefix));
|
||||
if (depth === 0) return;
|
||||
for (const character of alphabet) checkExhaustive(prefix + character, depth - 1);
|
||||
}
|
||||
checkExhaustive("", 3);
|
||||
|
||||
let seed = 0x12345678;
|
||||
function random(): number {
|
||||
seed = (seed * 1664525 + 1013904223) >>> 0;
|
||||
return seed / 0x100000000;
|
||||
}
|
||||
for (let i = 0; i < 1_000; i++) {
|
||||
let input = "";
|
||||
const length = Math.floor(random() * 80);
|
||||
for (let j = 0; j < length; j++) input += alphabet[Math.floor(random() * alphabet.length)];
|
||||
assertMatchesBufferTail(input, sampledByteLimits(input));
|
||||
}
|
||||
});
|
||||
});
|
||||
523
third_party/pi/packages/agent/test/harness/types.test.ts
vendored
Normal file
523
third_party/pi/packages/agent/test/harness/types.test.ts
vendored
Normal file
@@ -0,0 +1,523 @@
|
||||
import type { AssistantMessage, AssistantMessageFrame, DeferredHandle, Usage } from "@earendil-works/pi-ai";
|
||||
import { expectTypeOf, it } from "vitest";
|
||||
import * as storedValues from "../../src/harness/session/values.ts";
|
||||
import type {
|
||||
AgentHarness,
|
||||
AgentHarnessOptions,
|
||||
AgentHarnessStreamOptions,
|
||||
AgentHarnessTool,
|
||||
AgentHarnessToolInvocation,
|
||||
AgentLane,
|
||||
AgentMessage,
|
||||
AgentTool,
|
||||
Branch,
|
||||
BranchScan,
|
||||
CancelQueuedResult,
|
||||
CheckpointData,
|
||||
Context,
|
||||
Control,
|
||||
CustomEntry,
|
||||
DriveOptions,
|
||||
DriveOutcome,
|
||||
DriveResult,
|
||||
Entry,
|
||||
EntryProjector,
|
||||
EntryWrite,
|
||||
GenerationContext,
|
||||
HarnessEvent,
|
||||
HookHandler,
|
||||
HookMap,
|
||||
HookName,
|
||||
IdGenerator,
|
||||
InboxItem,
|
||||
LaneConfiguration,
|
||||
LaneExecutionInfo,
|
||||
LaneSnapshot,
|
||||
LaneSnapshotTool,
|
||||
NewEntry,
|
||||
OperationAdmissionResult,
|
||||
OperationAt,
|
||||
OperationMeta,
|
||||
OperationRequest,
|
||||
OperationResultRecord,
|
||||
OperationScope,
|
||||
OperationState,
|
||||
RunResult,
|
||||
SearchQuery,
|
||||
Session,
|
||||
SessionCreateOptions,
|
||||
SessionMetadata,
|
||||
SessionMutation,
|
||||
SessionMutator,
|
||||
SessionReader,
|
||||
SessionRepo,
|
||||
SessionSearchHit,
|
||||
SessionSearchService,
|
||||
SessionSnapshot,
|
||||
SessionStats,
|
||||
SettledAssistantMessage,
|
||||
Storage,
|
||||
StorageBranchScan,
|
||||
SummaryContext,
|
||||
SuspendedRun,
|
||||
TerminalStatus,
|
||||
ToolCall,
|
||||
UsageRow,
|
||||
UsageWrite,
|
||||
ValueSetWrite,
|
||||
Write,
|
||||
} from "../../src/index.ts";
|
||||
import { insertEntry, insertUsage } from "../../src/index.ts";
|
||||
|
||||
const configuration = {
|
||||
model: { provider: "provider", modelId: "model" },
|
||||
thinkingLevel: "off",
|
||||
activeToolNames: ["read"],
|
||||
} satisfies LaneConfiguration;
|
||||
|
||||
const retryPolicy = { maxAttempts: 3, baseDelayMs: 100 } as const;
|
||||
const generationContext = {
|
||||
stepId: "step",
|
||||
triggerEntryId: "trigger",
|
||||
configuration,
|
||||
streamOptions: { deferred: { window: "1h" } },
|
||||
retryPolicy,
|
||||
overflowRecoveryUsed: false,
|
||||
} satisfies GenerationContext;
|
||||
const summaryContext = {
|
||||
resultEntryId: "summary",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
retryPolicy,
|
||||
} satisfies SummaryContext;
|
||||
const checkpoint = {
|
||||
continuation: { kind: "need_assistant", overflowRecoveryUsed: false },
|
||||
triggerEntryId: "trigger",
|
||||
} satisfies CheckpointData;
|
||||
|
||||
const runningControl = { status: "running" } satisfies Control;
|
||||
const toolCalls = [
|
||||
{ status: "planned", sourceIndex: 0, resultEntryId: "result-0" },
|
||||
{ status: "effect_pending", sourceIndex: 1, resultEntryId: "result-1", replay: "safe" },
|
||||
{ status: "outcome_ready", sourceIndex: 2, resultEntryId: "result-2", terminate: true },
|
||||
{ status: "completed", sourceIndex: 3, resultEntryId: "result-3", terminate: false },
|
||||
] satisfies ToolCall[];
|
||||
|
||||
const runScope = {
|
||||
control: runningControl,
|
||||
settings: {
|
||||
compaction: { enabled: true, reserveTokens: 1000, keepRecentTokens: 2000 },
|
||||
steeringMode: "all",
|
||||
followUpMode: "one-at-a-time",
|
||||
toolExecution: "parallel",
|
||||
},
|
||||
latestAssistantEntryId: null,
|
||||
} satisfies OperationScope;
|
||||
|
||||
const runState = { ...runScope, at: "starting" } satisfies OperationState;
|
||||
// One value fixture per flat leaf; `satisfies OperationState[]` keeps every discriminant checked.
|
||||
const operationStates = [
|
||||
runState,
|
||||
{ ...runScope, at: "checkpoint", ...checkpoint },
|
||||
{ ...runScope, at: "assistant.ready", generationContext, nextAttempt: 1 },
|
||||
{
|
||||
...runScope,
|
||||
at: "assistant.effect_pending",
|
||||
generationContext,
|
||||
attempt: 1,
|
||||
responseEntryId: "response",
|
||||
usageId: "usage",
|
||||
intendedOutputLimit: 4096,
|
||||
contextWindow: 128000,
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "assistant.retry_wait",
|
||||
generationContext,
|
||||
nextAttempt: 2,
|
||||
notBefore: 10,
|
||||
errorMessage: "retry",
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "tools",
|
||||
batch: { assistantEntryId: "assistant", configuration, turnId: "turn", calls: toolCalls },
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "deferred.suspended",
|
||||
stepId: "step",
|
||||
sourceEntryId: "source",
|
||||
poll: 0,
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "deferred.effect_pending",
|
||||
stepId: "step",
|
||||
sourceEntryId: "source",
|
||||
poll: 1,
|
||||
responseEntryId: "response",
|
||||
usageId: "usage",
|
||||
configuration,
|
||||
streamOptions: {},
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "summary.deciding",
|
||||
task: { taskId: "task", reason: "threshold", boundary: { kind: "resume_checkpoint", resumeAfter: checkpoint } },
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "summary.ready",
|
||||
task: { taskId: "task", reason: "manual", customInstructions: "compact", boundary: { kind: "finish" } },
|
||||
summaryContext,
|
||||
nextAttempt: 1,
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "summary.effect_pending",
|
||||
task: {
|
||||
taskId: "task",
|
||||
boundary: { kind: "commit_navigation", targetId: "target", label: "target" },
|
||||
},
|
||||
summaryContext,
|
||||
attempt: 1,
|
||||
request: { index: 0, usageId: "usage" },
|
||||
usageIds: [],
|
||||
},
|
||||
{
|
||||
...runScope,
|
||||
at: "summary.retry_wait",
|
||||
task: { taskId: "task", reason: "overflow", boundary: { kind: "resume_checkpoint", resumeAfter: checkpoint } },
|
||||
summaryContext,
|
||||
nextAttempt: 2,
|
||||
notBefore: 10,
|
||||
errorMessage: "retry",
|
||||
},
|
||||
{ ...runScope, at: "navigation.ready_to_commit", targetId: null },
|
||||
] satisfies OperationState[];
|
||||
const operations = [
|
||||
{
|
||||
operationId: "run",
|
||||
lane: "main",
|
||||
sourceTipId: null,
|
||||
startedAt: 1,
|
||||
intent: { kind: "run", promptEntryIds: ["prompt"] },
|
||||
},
|
||||
{
|
||||
operationId: "compaction",
|
||||
lane: "main",
|
||||
sourceTipId: "source",
|
||||
startedAt: 2,
|
||||
intent: { kind: "compaction", customInstructions: "compact" },
|
||||
},
|
||||
{
|
||||
operationId: "navigation",
|
||||
lane: "main",
|
||||
sourceTipId: "source",
|
||||
startedAt: 3,
|
||||
intent: { kind: "navigation", targetId: "target", summarize: true, label: "target" },
|
||||
},
|
||||
] satisfies OperationMeta[];
|
||||
|
||||
const operationResult = {
|
||||
operationId: "run",
|
||||
kind: "run",
|
||||
status: "completed",
|
||||
fromTipId: null,
|
||||
tipId: "leaf",
|
||||
startedAt: 1,
|
||||
endedAt: 2,
|
||||
} satisfies OperationResultRecord;
|
||||
const valueWrites: ValueSetWrite[] = [
|
||||
storedValues.setValue(storedValues.branchTip("main"), "leaf"),
|
||||
storedValues.setValue(storedValues.laneConfig("main"), configuration),
|
||||
storedValues.setValue(storedValues.laneState("main"), {
|
||||
currentOperationId: "run",
|
||||
lastOperationId: null,
|
||||
inbox: [],
|
||||
}),
|
||||
storedValues.setValue(storedValues.operationResult("run"), operationResult),
|
||||
storedValues.setValue(storedValues.operationMeta("run"), operations[0]),
|
||||
storedValues.setValue(storedValues.operationState("run"), runState),
|
||||
storedValues.setValue(storedValues.operationToolArgs("run", "step", 0), { path: "file" }),
|
||||
storedValues.setValue(storedValues.operationPreparation("run", "task"), {
|
||||
kind: "compaction",
|
||||
messagesToSummarize: [],
|
||||
turnPrefixMessages: [],
|
||||
retainedTail: [],
|
||||
isSplitTurn: false,
|
||||
tokensBefore: 100,
|
||||
fileOps: { read: [], written: [], edited: [] },
|
||||
settings: { enabled: true, reserveTokens: 1000, keepRecentTokens: 2000 },
|
||||
}),
|
||||
storedValues.setValue(storedValues.pendingEntry("pending"), {
|
||||
type: "custom",
|
||||
customType: "note",
|
||||
payload: { text: "pending" },
|
||||
}),
|
||||
storedValues.setValue(storedValues.sessionName, "session"),
|
||||
storedValues.setValue(storedValues.entryLabel("entry"), "label"),
|
||||
storedValues.setValue(storedValues.value<unknown>("test.value", "state"), null),
|
||||
];
|
||||
|
||||
const usage = {
|
||||
input: 1,
|
||||
output: 2,
|
||||
cacheRead: 3,
|
||||
cacheWrite: 4,
|
||||
totalTokens: 10,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
} satisfies Usage;
|
||||
const usageRow = {
|
||||
id: "usage",
|
||||
seq: 2,
|
||||
usage,
|
||||
entryId: "entry",
|
||||
adjustment: false,
|
||||
details: { attempt: 1 },
|
||||
} satisfies UsageRow;
|
||||
const entryWrite = insertEntry({
|
||||
id: "entry",
|
||||
parentId: null,
|
||||
type: "message",
|
||||
message: { role: "user", content: "hello", timestamp: 1 },
|
||||
});
|
||||
const usageWrite = insertUsage({
|
||||
id: usageRow.id,
|
||||
usage,
|
||||
adjustment: false,
|
||||
entryId: "entry",
|
||||
});
|
||||
const writes = [
|
||||
entryWrite,
|
||||
usageWrite,
|
||||
valueWrites[0]!,
|
||||
storedValues.deleteValue(storedValues.entryLabel("entry")),
|
||||
] satisfies Write[];
|
||||
const transaction = writes satisfies Write[];
|
||||
|
||||
it("covers the complete durable storage and Part 3 discriminants", () => {
|
||||
expectTypeOf(entryWrite).toEqualTypeOf<EntryWrite>();
|
||||
expectTypeOf(usageWrite).toEqualTypeOf<UsageWrite>();
|
||||
expectTypeOf(storedValues.laneConfig("main")).toEqualTypeOf<storedValues.Value<LaneConfiguration>>();
|
||||
expectTypeOf(storedValues.pendingAssistantFrames("operation", "response")).toEqualTypeOf<
|
||||
storedValues.ValueList<AssistantMessageFrame>
|
||||
>();
|
||||
expectTypeOf<OperationMeta["intent"]["kind"]>().toEqualTypeOf<"run" | "compaction" | "navigation">();
|
||||
expectTypeOf<Control["status"]>().toEqualTypeOf<"running" | "cancel_requested">();
|
||||
expectTypeOf<ToolCall["status"]>().toEqualTypeOf<"planned" | "effect_pending" | "outcome_ready" | "completed">();
|
||||
expectTypeOf<OperationAt>().toEqualTypeOf<
|
||||
| "starting"
|
||||
| "checkpoint"
|
||||
| "assistant.ready"
|
||||
| "assistant.effect_pending"
|
||||
| "assistant.retry_wait"
|
||||
| "tools"
|
||||
| "deferred.suspended"
|
||||
| "deferred.effect_pending"
|
||||
| "summary.deciding"
|
||||
| "summary.ready"
|
||||
| "summary.effect_pending"
|
||||
| "summary.retry_wait"
|
||||
| "summary.deciding"
|
||||
| "summary.ready"
|
||||
| "summary.effect_pending"
|
||||
| "summary.retry_wait"
|
||||
| "navigation.ready_to_commit"
|
||||
| "summary.deciding"
|
||||
| "summary.ready"
|
||||
| "summary.effect_pending"
|
||||
| "summary.retry_wait"
|
||||
>();
|
||||
expectTypeOf<InboxItem["kind"]>().toEqualTypeOf<"steer" | "followUp" | "nextRun" | "write">();
|
||||
expectTypeOf<TerminalStatus>().toEqualTypeOf<"completed" | "declined" | "aborted" | "failed">();
|
||||
expectTypeOf<NewEntry["type"]>().toEqualTypeOf<"message" | "compaction" | "branch_summary" | "custom">();
|
||||
void transaction;
|
||||
void operationStates;
|
||||
|
||||
const compileTimeFailures = () => {
|
||||
// @ts-expect-error lane.config requires a complete LaneConfiguration
|
||||
const invalidValue: ValueSetWrite = storedValues.setValue(storedValues.laneConfig("main"), "model");
|
||||
// @ts-expect-error response entries require settled assistant content at runtime, not a pending settlement type
|
||||
const invalidSettled: SettledAssistantMessage = { stopReason: "pending" } as AssistantMessage;
|
||||
void invalidValue;
|
||||
void invalidSettled;
|
||||
};
|
||||
expectTypeOf(compileTimeFailures).toBeFunction();
|
||||
});
|
||||
|
||||
it("covers storage, session, repository, search, and identity signatures", () => {
|
||||
expectTypeOf<SessionMetadata["storageVersion"]>().toEqualTypeOf<number>();
|
||||
expectTypeOf<IdGenerator["next"]>().toEqualTypeOf<(timestampMs?: number) => string>();
|
||||
expectTypeOf<Parameters<Storage["scanBranch"]>[0]["start"]>().toEqualTypeOf<string>();
|
||||
expectTypeOf<BranchScan["start"]>().toEqualTypeOf<string | undefined>();
|
||||
expectTypeOf<Storage["commit"]>().toEqualTypeOf<
|
||||
(
|
||||
transactionToCommit: Write[],
|
||||
context: Context,
|
||||
) => Promise<{ firstSeq: number; seqs: number[]; timestamp: number; stats: SessionStats }>
|
||||
>();
|
||||
expectTypeOf<Session["beginMutation"]>().toEqualTypeOf<(context: Context) => Promise<SessionMutation>>();
|
||||
expectTypeOf<SessionMutation["commit"]>().toEqualTypeOf<SessionMutator["commit"]>();
|
||||
expectTypeOf<SessionMutation["end"]>().toEqualTypeOf<(context: Context) => Promise<void>>();
|
||||
expectTypeOf<Session["mutate"]>().toEqualTypeOf<
|
||||
<T>(mutation: (mutator: SessionMutator, context: Context) => T | Promise<T>, context: Context) => Promise<T>
|
||||
>();
|
||||
expectTypeOf<SessionMutator["commit"]>().toEqualTypeOf<
|
||||
(
|
||||
transactionToCommit: Write[],
|
||||
context: Context,
|
||||
) => Promise<{ firstSeq: number; seqs: number[]; timestamp: number; stats: SessionStats }>
|
||||
>();
|
||||
expectTypeOf<SessionReader["scanBranch"]>().toEqualTypeOf<
|
||||
(query: StorageBranchScan, context: Context) => Promise<Entry[]>
|
||||
>();
|
||||
expectTypeOf<Session["createBranch"]>().toEqualTypeOf<
|
||||
(name: string, at: string | null, context: Context) => Promise<Branch>
|
||||
>();
|
||||
expectTypeOf<SessionRepo["create"]>().toEqualTypeOf<
|
||||
(options: SessionCreateOptions, context: Context) => Promise<Session>
|
||||
>();
|
||||
expectTypeOf<SessionSearchService["searchSessions"]>().toEqualTypeOf<
|
||||
(query: SearchQuery) => Promise<SessionSearchHit[]>
|
||||
>();
|
||||
expectTypeOf<SessionSearchService["notify"]>().toEqualTypeOf<(sessionId: string) => void>();
|
||||
});
|
||||
|
||||
it("covers Part 5 results, events, hooks, snapshots, tools, and stream options", () => {
|
||||
type RunErrorTag = Extract<RunResult, { ok: false }>["error"]["_tag"];
|
||||
type CancelKind = Extract<CancelQueuedResult, { ok: true }>["value"]["kind"];
|
||||
expectTypeOf<RunErrorTag>().toEqualTypeOf<
|
||||
"LaneBusy" | "InvalidMessage" | "UnknownSkill" | "UnknownTemplate" | "Closed"
|
||||
>();
|
||||
expectTypeOf<CancelKind>().toEqualTypeOf<"cancelled" | "already_consumed" | "not_found">();
|
||||
expectTypeOf<HarnessEvent["type"]>().toEqualTypeOf<
|
||||
| "run_start"
|
||||
| "run_resume"
|
||||
| "run_suspend"
|
||||
| "operation_abort"
|
||||
| "run_end"
|
||||
| "fault"
|
||||
| "handler_error"
|
||||
| "turn_start"
|
||||
| "turn_end"
|
||||
| "retry_scheduled"
|
||||
| "retry_start"
|
||||
| "retry_end"
|
||||
| "message_start"
|
||||
| "message_update"
|
||||
| "message_end"
|
||||
| "tool_start"
|
||||
| "tool_update"
|
||||
| "tool_end"
|
||||
| "entry_added"
|
||||
| "queue_update"
|
||||
| "value_update"
|
||||
| "config_update"
|
||||
| "compaction_start"
|
||||
| "compaction_end"
|
||||
| "navigation_start"
|
||||
| "navigation_end"
|
||||
| "lane_created"
|
||||
| "usage"
|
||||
>();
|
||||
expectTypeOf<HookName>().toEqualTypeOf<
|
||||
| "before_run"
|
||||
| "before_drive"
|
||||
| "before_run_end"
|
||||
| "transform_context"
|
||||
| "before_request"
|
||||
| "before_payload"
|
||||
| "after_response"
|
||||
| "before_tool"
|
||||
| "after_tool"
|
||||
| "before_compaction"
|
||||
| "before_navigation"
|
||||
>();
|
||||
expectTypeOf<HookMap["before_drive"]["result"]>().toEqualTypeOf<void>();
|
||||
expectTypeOf<HookHandler<"before_drive">>().returns.toEqualTypeOf<void | Promise<void>>();
|
||||
expectTypeOf<HookMap["transform_context"]["event"]>().toEqualTypeOf<{
|
||||
messages: AgentMessage[];
|
||||
systemPrompt: string;
|
||||
}>();
|
||||
expectTypeOf<LaneSnapshot["operation"]>().not.toEqualTypeOf<SessionSnapshot>();
|
||||
expectTypeOf<LaneSnapshotTool["status"]>().toEqualTypeOf<"running" | "settled">();
|
||||
expectTypeOf<AgentLane["getResult"]>().returns.toEqualTypeOf<Promise<OperationResultRecord | undefined>>();
|
||||
expectTypeOf<SuspendedRun>().toEqualTypeOf<{
|
||||
operationId: string;
|
||||
status: "suspended";
|
||||
deferred: DeferredHandle;
|
||||
}>();
|
||||
expectTypeOf<AgentLane["accept"]>().returns.toEqualTypeOf<Promise<OperationAdmissionResult>>();
|
||||
expectTypeOf<AgentLane["drive"]>().returns.toEqualTypeOf<Promise<DriveResult>>();
|
||||
expectTypeOf<keyof DriveOptions>().toEqualTypeOf<"operationId" | "waitForRetry" | "pollDeferred">();
|
||||
expectTypeOf<DriveOutcome["kind"]>().toEqualTypeOf<"settled" | "waiting">();
|
||||
expectTypeOf<AgentLane["inspectExecution"]>().returns.toEqualTypeOf<Promise<LaneExecutionInfo>>();
|
||||
expectTypeOf<AgentLane["getTipId"]>().returns.toEqualTypeOf<Promise<string | null>>();
|
||||
expectTypeOf<Extract<keyof AgentLane, "sessionTree">>().toEqualTypeOf<never>();
|
||||
expectTypeOf<Extract<keyof AgentHarness, keyof AgentLane>>().toEqualTypeOf<never>();
|
||||
expectTypeOf<
|
||||
Extract<
|
||||
keyof AgentLane,
|
||||
| "session"
|
||||
| "models"
|
||||
| "hooks"
|
||||
| "activeDrive"
|
||||
| "command"
|
||||
| "settleOperation"
|
||||
| "continueOperation"
|
||||
| "readConfig"
|
||||
| "mismatch"
|
||||
>
|
||||
>().toEqualTypeOf<never>();
|
||||
expectTypeOf<
|
||||
Extract<
|
||||
keyof AgentHarness,
|
||||
| "session"
|
||||
| "models"
|
||||
| "activeDrive"
|
||||
| "command"
|
||||
| "settleOperation"
|
||||
| "continueOperation"
|
||||
| "readConfig"
|
||||
| "mismatch"
|
||||
>
|
||||
>().toEqualTypeOf<never>();
|
||||
expectTypeOf<OperationRequest["kind"]>().toEqualTypeOf<
|
||||
"prompt" | "skill" | "prompt_template" | "compaction" | "navigation"
|
||||
>();
|
||||
expectTypeOf<Parameters<AgentHarnessTool<object>["execute"]>[4]>().toEqualTypeOf<AgentHarnessToolInvocation>();
|
||||
expectTypeOf<Parameters<AgentHarnessTool<object>["execute"]>[5]>().toEqualTypeOf<Context>();
|
||||
expectTypeOf<AgentTool["replay"]>().toEqualTypeOf<"never" | "safe" | undefined>();
|
||||
expectTypeOf<AgentHarnessStreamOptions["deferred"]>().toEqualTypeOf<
|
||||
boolean | { window?: "15m" | "1h" | "24h" } | undefined
|
||||
>();
|
||||
expectTypeOf<EntryProjector>().toEqualTypeOf<
|
||||
(entry: CustomEntry, context: Context) => AgentMessage[] | undefined | Promise<AgentMessage[] | undefined>
|
||||
>();
|
||||
expectTypeOf<NonNullable<AgentHarnessOptions["entryProjectors"]>>().toEqualTypeOf<Record<string, EntryProjector>>();
|
||||
|
||||
const compileTimeFailures = () => {
|
||||
// @ts-expect-error callers cannot supply the harness-owned abort signal
|
||||
const invalidOptions: AgentHarnessStreamOptions = { signal: new AbortController().signal };
|
||||
const invalidDriveOptions: DriveOptions = {
|
||||
operationId: "run",
|
||||
// @ts-expect-error drive has no wall-clock deadline
|
||||
deadline: Date.now(),
|
||||
};
|
||||
const invalidValueEvent: Extract<HarnessEvent, { type: "value_update" }> = {
|
||||
type: "value_update",
|
||||
value: "session_name",
|
||||
name: "session",
|
||||
// @ts-expect-error value events are harness-global and cannot carry a lane
|
||||
lane: "main",
|
||||
};
|
||||
void invalidOptions;
|
||||
void invalidDriveOptions;
|
||||
void invalidValueEvent;
|
||||
};
|
||||
expectTypeOf(compileTimeFailures).toBeFunction();
|
||||
});
|
||||
183
third_party/pi/packages/agent/test/harness/values.test.ts
vendored
Normal file
183
third_party/pi/packages/agent/test/harness/values.test.ts
vendored
Normal file
@@ -0,0 +1,183 @@
|
||||
import type { AssistantMessageFrame } from "@earendil-works/pi-ai";
|
||||
import { describe, expect, expectTypeOf, it } from "vitest";
|
||||
import { BACKGROUND_CONTEXT } from "../../src/harness/context.ts";
|
||||
import { MemoryStorage } from "../../src/harness/session/memory.ts";
|
||||
import type {
|
||||
DurableStructuralPreparation,
|
||||
JsonValue,
|
||||
LaneConfiguration,
|
||||
LaneState,
|
||||
OperationMeta,
|
||||
OperationResultRecord,
|
||||
OperationState,
|
||||
PendingEntry,
|
||||
SessionReader,
|
||||
} from "../../src/harness/session/types.ts";
|
||||
import {
|
||||
appendList,
|
||||
branchTip,
|
||||
branchTipInventoryPrefix,
|
||||
deleteList,
|
||||
deleteValue,
|
||||
entryLabel,
|
||||
type ListElement,
|
||||
laneConfig,
|
||||
laneState,
|
||||
list,
|
||||
operationMeta,
|
||||
operationPreparation,
|
||||
operationPreparationPrefix,
|
||||
operationResult,
|
||||
operationState,
|
||||
operationToolArgs,
|
||||
operationToolArgsPrefix,
|
||||
operationToolMemo,
|
||||
operationToolMemoPrefix,
|
||||
pendingAssistantFrames,
|
||||
pendingEntry,
|
||||
pendingToolOutput,
|
||||
pendingToolOutputPrefix,
|
||||
type StoredValue,
|
||||
sessionName,
|
||||
setValue,
|
||||
type Value,
|
||||
type ValueList,
|
||||
value,
|
||||
} from "../../src/harness/session/values.ts";
|
||||
|
||||
describe("bound value addresses", () => {
|
||||
it("binds immutable scalar and list addresses with validated components", () => {
|
||||
expect(value<string>("app.state")).toEqual({ kind: "value", namespace: "app.state", key: "" });
|
||||
expect(list<number>("app.events", "workspace")).toEqual({
|
||||
kind: "list",
|
||||
namespace: "app.events",
|
||||
key: "workspace",
|
||||
});
|
||||
expect(Object.isFrozen(value<string>("app.state"))).toBe(true);
|
||||
expect(() => value<unknown>("")).toThrow("must not be empty");
|
||||
expect(() => value<unknown>("app\0state")).toThrow("must not contain");
|
||||
expect(() => list<unknown>("app.events", "bad\0key")).toThrow("must not contain");
|
||||
expect(value<unknown>("pi.application")).toEqual({ kind: "value", namespace: "pi.application", key: "" });
|
||||
});
|
||||
|
||||
it("preserves invariant address and helper types", () => {
|
||||
const scalar = value<{ count: number }>("app.state");
|
||||
const events = list<{ name: string }>("app.events");
|
||||
expectTypeOf(scalar).toEqualTypeOf<Value<{ count: number }>>();
|
||||
expectTypeOf(events).toEqualTypeOf<ValueList<{ name: string }>>();
|
||||
expectTypeOf(setValue(scalar, { count: 1 }).value).toEqualTypeOf<unknown>();
|
||||
expectTypeOf(appendList(events, { name: "created" }).value).toEqualTypeOf<unknown>();
|
||||
|
||||
const compileTimeFailures = () => {
|
||||
// @ts-expect-error Value is invariant in its stored type
|
||||
const widenedScalar: Value<unknown> = scalar;
|
||||
// @ts-expect-error ValueList is invariant in its element type
|
||||
const widenedList: ValueList<unknown> = events;
|
||||
// @ts-expect-error the bound scalar address controls the write type
|
||||
setValue(scalar, { count: "wrong" });
|
||||
// @ts-expect-error the bound list address controls the element type
|
||||
appendList(events, { name: 1 });
|
||||
// @ts-expect-error scalar helpers reject list addresses
|
||||
deleteValue(events);
|
||||
// @ts-expect-error list helpers reject scalar addresses
|
||||
deleteList(scalar);
|
||||
void widenedScalar;
|
||||
void widenedList;
|
||||
};
|
||||
expectTypeOf(compileTimeFailures).toBeFunction();
|
||||
});
|
||||
|
||||
it("infers list element types through Storage and SessionReader", async () => {
|
||||
const storage = new MemoryStorage({ now: () => 1 });
|
||||
const events = list<{ name: string }>("app.events");
|
||||
const reader: SessionReader = storage;
|
||||
const storageElements = await storage.readList(events, undefined, BACKGROUND_CONTEXT);
|
||||
const readerElements = await reader.readList(events, undefined, BACKGROUND_CONTEXT);
|
||||
expectTypeOf(storageElements).toEqualTypeOf<ListElement<{ name: string }>[]>();
|
||||
expectTypeOf(readerElements).toEqualTypeOf<ListElement<{ name: string }>[]>();
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
|
||||
it("uses separately constructed equal addresses for one durable location", async () => {
|
||||
const storage = new MemoryStorage({ now: () => 1 });
|
||||
const first = value<{ ready: boolean }>("app.state", "workspace");
|
||||
const second = value<{ ready: boolean }>("app.state", "workspace");
|
||||
await storage.commit([setValue(first, { ready: true })], BACKGROUND_CONTEXT);
|
||||
const stored = await storage.getValue(second, BACKGROUND_CONTEXT);
|
||||
expect(stored).toEqual({ address: first, value: { ready: true }, seq: 1 });
|
||||
expectTypeOf(stored).toEqualTypeOf<StoredValue<{ ready: boolean }> | undefined>();
|
||||
await storage.close(BACKGROUND_CONTEXT);
|
||||
});
|
||||
});
|
||||
|
||||
describe("built-in durable addresses", () => {
|
||||
it("uses the exact reserved namespaces, keys, kinds, and value types", () => {
|
||||
const config = laneConfig("review");
|
||||
const meta = operationMeta("operation");
|
||||
const state = operationState("operation");
|
||||
const frames = pendingAssistantFrames("operation", "response");
|
||||
expectTypeOf(config).toEqualTypeOf<Value<LaneConfiguration>>();
|
||||
expectTypeOf(branchTip("review")).toEqualTypeOf<Value<string | null>>();
|
||||
expectTypeOf(laneState("review")).toEqualTypeOf<Value<LaneState>>();
|
||||
expectTypeOf(operationResult("operation")).toEqualTypeOf<Value<OperationResultRecord>>();
|
||||
expectTypeOf(meta).toEqualTypeOf<Value<OperationMeta>>();
|
||||
expectTypeOf(state).toEqualTypeOf<Value<OperationState>>();
|
||||
expectTypeOf(operationToolArgs("operation", "step", 2)).toEqualTypeOf<Value<Record<string, JsonValue>>>();
|
||||
expectTypeOf(operationToolMemo("operation", "invocation", "name")).toEqualTypeOf<Value<JsonValue>>();
|
||||
expectTypeOf(operationPreparation("operation", "task")).toEqualTypeOf<Value<DurableStructuralPreparation>>();
|
||||
expectTypeOf(pendingEntry("entry")).toEqualTypeOf<Value<PendingEntry>>();
|
||||
expectTypeOf(frames).toEqualTypeOf<ValueList<AssistantMessageFrame>>();
|
||||
|
||||
expect([
|
||||
branchTip("review"),
|
||||
laneConfig("review"),
|
||||
laneState("review"),
|
||||
operationResult("operation"),
|
||||
meta,
|
||||
state,
|
||||
operationToolArgs("operation", "step", 2),
|
||||
operationToolMemo("operation", "invocation", "name"),
|
||||
operationPreparation("operation", "task"),
|
||||
pendingEntry("entry"),
|
||||
pendingToolOutput("operation", "invocation"),
|
||||
frames,
|
||||
sessionName,
|
||||
entryLabel("entry"),
|
||||
]).toEqual([
|
||||
{ kind: "value", namespace: "pi.branch.tip", key: "review" },
|
||||
{ kind: "value", namespace: "pi.lane.config", key: "review" },
|
||||
{ kind: "value", namespace: "pi.lane.state", key: "review" },
|
||||
{ kind: "value", namespace: "pi.result", key: "operation" },
|
||||
{ kind: "value", namespace: "pi.op.meta", key: "operation" },
|
||||
{ kind: "value", namespace: "pi.op.state", key: "operation" },
|
||||
{ kind: "value", namespace: "pi.op.tool_args", key: "operation:step:2" },
|
||||
{ kind: "value", namespace: "pi.op.tool_memo", key: "operation:invocation:name" },
|
||||
{ kind: "value", namespace: "pi.op.preparation", key: "operation:task" },
|
||||
{ kind: "value", namespace: "pi.pending.entry", key: "entry" },
|
||||
{ kind: "value", namespace: "pi.pending.tool_output", key: "operation:invocation" },
|
||||
{ kind: "list", namespace: "pi.pending.assistant_frame", key: "operation:response" },
|
||||
{ kind: "value", namespace: "pi.session.name", key: "" },
|
||||
{ kind: "value", namespace: "pi.entry.label", key: "entry" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("exports exactly the five documented scan-prefix constructors", () => {
|
||||
const prefixes = [
|
||||
branchTipInventoryPrefix(),
|
||||
operationToolArgsPrefix("operation"),
|
||||
operationToolMemoPrefix("operation", "invocation"),
|
||||
operationPreparationPrefix("operation"),
|
||||
pendingToolOutputPrefix("operation"),
|
||||
];
|
||||
expect(prefixes).toHaveLength(5);
|
||||
expect(prefixes).toEqual([
|
||||
{ kind: "value", namespace: "pi.branch.tip", key: "" },
|
||||
{ kind: "value", namespace: "pi.op.tool_args", key: "operation:" },
|
||||
{ kind: "value", namespace: "pi.op.tool_memo", key: "operation:invocation:" },
|
||||
{ kind: "value", namespace: "pi.op.preparation", key: "operation:" },
|
||||
{ kind: "value", namespace: "pi.pending.tool_output", key: "operation:" },
|
||||
]);
|
||||
expect(operationToolArgsPrefix("operation", "step").key).toBe("operation:step:");
|
||||
expect(operationToolMemoPrefix("operation").key).toBe("operation:");
|
||||
});
|
||||
});
|
||||
129
third_party/pi/packages/agent/test/proxy.test.ts
vendored
Normal file
129
third_party/pi/packages/agent/test/proxy.test.ts
vendored
Normal file
@@ -0,0 +1,129 @@
|
||||
import type { AssistantMessage, AssistantMessageEvent, Model } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { type ProxyAssistantMessageEvent, streamProxy } from "../src/proxy.ts";
|
||||
|
||||
const model: Model<"openai-responses"> = {
|
||||
id: "gpt-5.4",
|
||||
name: "GPT-5.4",
|
||||
api: "openai-responses",
|
||||
provider: "openai",
|
||||
baseUrl: "https://api.openai.com/v1",
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 400000,
|
||||
maxTokens: 128000,
|
||||
};
|
||||
|
||||
const usage: AssistantMessage["usage"] = {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
};
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals();
|
||||
});
|
||||
|
||||
describe("streamProxy", () => {
|
||||
it("preserves tool-call metadata received only on toolcall_end", async () => {
|
||||
const proxyEvents: ProxyAssistantMessageEvent[] = [
|
||||
{ type: "start" },
|
||||
{ type: "toolcall_start", contentIndex: 0, id: "call_test|fc_test", toolName: "lookup" },
|
||||
{ type: "toolcall_delta", contentIndex: 0, delta: '{"value":"hello"}' },
|
||||
{
|
||||
type: "toolcall_end",
|
||||
contentIndex: 0,
|
||||
toolCall: {
|
||||
type: "toolCall",
|
||||
id: "call_test|fc_test",
|
||||
name: "lookup",
|
||||
arguments: { value: "hello" },
|
||||
namespace: "dynamic_tools",
|
||||
},
|
||||
},
|
||||
{ type: "done", reason: "toolUse", usage },
|
||||
];
|
||||
const body = proxyEvents.map((event) => `data: ${JSON.stringify(event)}\n\n`).join("");
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(body, { status: 200 })),
|
||||
);
|
||||
|
||||
const stream = streamProxy(
|
||||
model,
|
||||
{ systemPrompt: "", messages: [] },
|
||||
{
|
||||
authToken: "test-token",
|
||||
proxyUrl: "https://proxy.example.com",
|
||||
},
|
||||
);
|
||||
const events: AssistantMessageEvent[] = [];
|
||||
for await (const event of stream) events.push(event);
|
||||
const result = await stream.result();
|
||||
const endEvent = events.find((event) => event.type === "toolcall_end");
|
||||
|
||||
expect(endEvent).toMatchObject({
|
||||
type: "toolcall_end",
|
||||
toolCall: { namespace: "dynamic_tools" },
|
||||
});
|
||||
expect(result.content[0]).toMatchObject({
|
||||
type: "toolCall",
|
||||
arguments: { value: "hello" },
|
||||
namespace: "dynamic_tools",
|
||||
});
|
||||
});
|
||||
|
||||
// Regression tests for https://github.com/earendil-works/pi/issues/8996
|
||||
it("processes terminal metadata when the event is not newline-terminated", async () => {
|
||||
const start = `data: ${JSON.stringify({ type: "start" })}\n\n`;
|
||||
const done = `data: ${JSON.stringify({ type: "done", reason: "stop", usage, providerThinkingLevel: "high" })}`;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(start + done, { status: 200 })),
|
||||
);
|
||||
|
||||
const stream = streamProxy(
|
||||
model,
|
||||
{ systemPrompt: "", messages: [] },
|
||||
{
|
||||
authToken: "test-token",
|
||||
proxyUrl: "https://proxy.example.com",
|
||||
},
|
||||
);
|
||||
const events: AssistantMessageEvent[] = [];
|
||||
for await (const event of stream) events.push(event);
|
||||
const result = await stream.result();
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual(["start", "done"]);
|
||||
expect(result.stopReason).toBe("stop");
|
||||
expect(result.providerThinkingLevel).toBe("high");
|
||||
});
|
||||
|
||||
it("emits an error instead of hanging when the stream ends without a terminal event", async () => {
|
||||
const body = `data: ${JSON.stringify({ type: "start" })}\n\n`;
|
||||
vi.stubGlobal(
|
||||
"fetch",
|
||||
vi.fn(async () => new Response(body, { status: 200 })),
|
||||
);
|
||||
|
||||
const stream = streamProxy(
|
||||
model,
|
||||
{ systemPrompt: "", messages: [] },
|
||||
{
|
||||
authToken: "test-token",
|
||||
proxyUrl: "https://proxy.example.com",
|
||||
},
|
||||
);
|
||||
const events: AssistantMessageEvent[] = [];
|
||||
for await (const event of stream) events.push(event);
|
||||
const result = await stream.result();
|
||||
|
||||
expect(events.map((event) => event.type)).toEqual(["start", "error"]);
|
||||
expect(result.stopReason).toBe("error");
|
||||
expect(result.errorMessage).toContain("Connection closed by proxy server");
|
||||
});
|
||||
});
|
||||
40
third_party/pi/packages/agent/test/utils/calculate.ts
vendored
Normal file
40
third_party/pi/packages/agent/test/utils/calculate.ts
vendored
Normal file
@@ -0,0 +1,40 @@
|
||||
import type { Usage } from "@earendil-works/pi-ai";
|
||||
import { type Static, Type } from "typebox";
|
||||
import type { AgentTool, AgentToolResult } from "../../src/types.ts";
|
||||
|
||||
export interface CalculateResult extends AgentToolResult<undefined> {
|
||||
content: Array<{ type: "text"; text: string }>;
|
||||
details: undefined;
|
||||
}
|
||||
|
||||
export function calculate(expression: string): CalculateResult {
|
||||
try {
|
||||
const result = new Function(`return ${expression}`)();
|
||||
return { content: [{ type: "text", text: `${expression} = ${result}` }], details: undefined };
|
||||
} catch (e: any) {
|
||||
throw new Error(e.message || String(e));
|
||||
}
|
||||
}
|
||||
|
||||
const calculateSchema = Type.Object({
|
||||
expression: Type.String({ description: "The mathematical expression to evaluate" }),
|
||||
});
|
||||
|
||||
type CalculateParams = Static<typeof calculateSchema>;
|
||||
|
||||
export const calculateTool: AgentTool<typeof calculateSchema, undefined> = {
|
||||
label: "Calculator",
|
||||
name: "calculate",
|
||||
description: "Evaluate mathematical expressions",
|
||||
parameters: calculateSchema,
|
||||
execute: async (_toolCallId: string, args: CalculateParams) => {
|
||||
return calculate(args.expression);
|
||||
},
|
||||
};
|
||||
|
||||
export function createCalculateToolWithUsage(usage: Usage): AgentTool<typeof calculateSchema, undefined> {
|
||||
return {
|
||||
...calculateTool,
|
||||
execute: async (_toolCallId: string, args: CalculateParams) => ({ ...calculate(args.expression), usage }),
|
||||
};
|
||||
}
|
||||
46
third_party/pi/packages/agent/test/utils/get-current-time.ts
vendored
Normal file
46
third_party/pi/packages/agent/test/utils/get-current-time.ts
vendored
Normal file
@@ -0,0 +1,46 @@
|
||||
import { type Static, Type } from "typebox";
|
||||
import type { AgentTool, AgentToolResult } from "../../src/types.ts";
|
||||
|
||||
export interface GetCurrentTimeResult extends AgentToolResult<{ utcTimestamp: number }> {}
|
||||
|
||||
export async function getCurrentTime(timezone?: string): Promise<GetCurrentTimeResult> {
|
||||
const date = new Date();
|
||||
if (timezone) {
|
||||
try {
|
||||
const timeStr = date.toLocaleString("en-US", {
|
||||
timeZone: timezone,
|
||||
dateStyle: "full",
|
||||
timeStyle: "long",
|
||||
});
|
||||
return {
|
||||
content: [{ type: "text", text: timeStr }],
|
||||
details: { utcTimestamp: date.getTime() },
|
||||
};
|
||||
} catch (_e) {
|
||||
throw new Error(`Invalid timezone: ${timezone}. Current UTC time: ${date.toISOString()}`);
|
||||
}
|
||||
}
|
||||
const timeStr = date.toLocaleString("en-US", { dateStyle: "full", timeStyle: "long" });
|
||||
return {
|
||||
content: [{ type: "text", text: timeStr }],
|
||||
details: { utcTimestamp: date.getTime() },
|
||||
};
|
||||
}
|
||||
|
||||
const getCurrentTimeSchema = Type.Object({
|
||||
timezone: Type.Optional(
|
||||
Type.String({ description: "Optional timezone (e.g., 'America/New_York', 'Europe/London')" }),
|
||||
),
|
||||
});
|
||||
|
||||
type GetCurrentTimeParams = Static<typeof getCurrentTimeSchema>;
|
||||
|
||||
export const getCurrentTimeTool: AgentTool<typeof getCurrentTimeSchema, { utcTimestamp: number }> = {
|
||||
label: "Current Time",
|
||||
name: "get_current_time",
|
||||
description: "Get the current date and time",
|
||||
parameters: getCurrentTimeSchema,
|
||||
execute: async (_toolCallId: string, args: GetCurrentTimeParams) => {
|
||||
return getCurrentTime(args.timezone);
|
||||
},
|
||||
};
|
||||
3
third_party/pi/packages/agent/test/utils/wait-for-tick.ts
vendored
Normal file
3
third_party/pi/packages/agent/test/utils/wait-for-tick.ts
vendored
Normal file
@@ -0,0 +1,3 @@
|
||||
export function waitForTick(): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
Reference in New Issue
Block a user