chore(vendor): import pi 1.3.0
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
@@ -223,15 +223,18 @@ describe("AgentSession bash and persistence characterization", () => {
|
||||
await harness.session.prompt("start");
|
||||
|
||||
const entries = harness.sessionManager.getEntries();
|
||||
// The prompt is declared by the first request, after the queued custom message.
|
||||
expect(entries.map((entry) => entry.type)).toEqual([
|
||||
"custom_message",
|
||||
"message",
|
||||
"message",
|
||||
"message",
|
||||
"message",
|
||||
"message",
|
||||
]);
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual([
|
||||
"custom",
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
@@ -334,4 +337,63 @@ describe("AgentSession bash and persistence characterization", () => {
|
||||
{ id: "bash-1", delta: "world" },
|
||||
]);
|
||||
});
|
||||
|
||||
// Regression tests for https://github.com/earendil-works/pi/issues/10504
|
||||
describe("escape sequences split across output chunks", () => {
|
||||
/** Runs a user bash command whose fake shell emits the given chunks; `beforeExit` sees what was streamed by then. */
|
||||
async function runChunks(chunks: Array<string | Buffer>, beforeExit?: (streamed: string) => void) {
|
||||
const harness = await createHarness();
|
||||
harnesses.push(harness);
|
||||
const deltas: string[] = [];
|
||||
const operations: BashOperations = {
|
||||
exec: async (_command, _cwd, options) => {
|
||||
for (const chunk of chunks) {
|
||||
options.onData(typeof chunk === "string" ? Buffer.from(chunk) : chunk);
|
||||
}
|
||||
beforeExit?.(deltas.join(""));
|
||||
return { exitCode: 0 };
|
||||
},
|
||||
};
|
||||
const result = await harness.session.executeBash("custom", (delta) => deltas.push(delta), { operations });
|
||||
const recorded = harness.session.messages[harness.session.messages.length - 1];
|
||||
return {
|
||||
output: result.output,
|
||||
streamed: deltas.join(""),
|
||||
recorded: recorded?.role === "bashExecution" ? recorded.output : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
it("strips a color reset split inside its parameters", async () => {
|
||||
const { output, streamed, recorded } = await runChunks(["\x1b[31mERROR: file.py:1\x1b[0", "m\n"]);
|
||||
expect(output).toBe("ERROR: file.py:1\n");
|
||||
expect(streamed).toBe("ERROR: file.py:1\n");
|
||||
expect(recorded).toBe("ERROR: file.py:1\n");
|
||||
});
|
||||
|
||||
it("strips a color code split right after ESC", async () => {
|
||||
const { output, streamed } = await runChunks(["before\x1b", "[32mafter\n"]);
|
||||
expect(output).toBe("beforeafter\n");
|
||||
expect(streamed).toBe("beforeafter\n");
|
||||
});
|
||||
|
||||
it("strips an OSC sequence split before its terminator", async () => {
|
||||
const { output } = await runChunks(["a\x1b]0;window ", "title\x1b", "\\b\n"]);
|
||||
expect(output).toBe("ab\n");
|
||||
});
|
||||
|
||||
it("flushes an incomplete multi-byte character at the end of output", async () => {
|
||||
const { output, streamed } = await runChunks(["ok", Buffer.from("\u00e9").subarray(0, 1)]);
|
||||
expect(output).toBe("ok\uFFFD");
|
||||
expect(streamed).toBe("ok\uFFFD");
|
||||
});
|
||||
|
||||
it("does not hold back output behind a long unterminated sequence", async () => {
|
||||
const long = "x".repeat(300);
|
||||
let streamedBeforeExit = "";
|
||||
await runChunks([`\x1b]${long}`], (streamed) => {
|
||||
streamedBeforeExit = streamed;
|
||||
});
|
||||
expect(streamedBeforeExit).toBe(`]${long}`);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
1063
third_party/pi/packages/coding-agent/test/suite/agent-session-boundaries.test.ts
vendored
Normal file
1063
third_party/pi/packages/coding-agent/test/suite/agent-session-boundaries.test.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
1032
third_party/pi/packages/coding-agent/test/suite/agent-session-codemode.test.ts
vendored
Normal file
1032
third_party/pi/packages/coding-agent/test/suite/agent-session-codemode.test.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
206
third_party/pi/packages/coding-agent/test/suite/agent-session-compaction-model-overrides.test.ts
vendored
Normal file
206
third_party/pi/packages/coding-agent/test/suite/agent-session-compaction-model-overrides.test.ts
vendored
Normal file
@@ -0,0 +1,206 @@
|
||||
import { fauxAssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { SessionBeforeCompactEvent } from "../../src/core/extensions/index.ts";
|
||||
import { createHarness, type Harness } from "./harness.ts";
|
||||
|
||||
function seedHistory(harness: Harness, totalTokens = 650): string {
|
||||
const model = harness.session.model!;
|
||||
let recentUserId = "";
|
||||
for (const label of ["old", "recent"]) {
|
||||
recentUserId = harness.sessionManager.appendMessage({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: label.padEnd(400, "x") }],
|
||||
timestamp: Date.now() - 2000,
|
||||
});
|
||||
const assistant = fauxAssistantMessage(label.padEnd(400, "y"), { timestamp: Date.now() - 1000 });
|
||||
harness.sessionManager.appendMessage({
|
||||
...assistant,
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: { ...assistant.usage, input: totalTokens, totalTokens },
|
||||
});
|
||||
}
|
||||
harness.session.agent.state.messages = harness.sessionManager.buildSessionContext().messages;
|
||||
return recentUserId;
|
||||
}
|
||||
|
||||
// Regression coverage for #8133.
|
||||
describe("AgentSession compaction model overrides", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
while (harnesses.length) harnesses.pop()?.cleanup();
|
||||
});
|
||||
|
||||
it.each(["manual", "pre-prompt", "post-run", "overflow"] as const)(
|
||||
"uses model token settings for %s compaction and extension preparation",
|
||||
async (path) => {
|
||||
const preparations: SessionBeforeCompactEvent[] = [];
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "faux-1", contextWindow: 4000 }],
|
||||
tools: [],
|
||||
settings: {
|
||||
compaction: {
|
||||
enabled: path !== "manual",
|
||||
reserveTokens: 10,
|
||||
keepRecentTokens: 20000,
|
||||
modelOverrides: { "faux/faux-1": { reserveTokens: 2000, keepRecentTokens: 150 } },
|
||||
},
|
||||
},
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", (event) => {
|
||||
preparations.push(event);
|
||||
return {
|
||||
compaction: {
|
||||
summary: "compacted history",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
},
|
||||
};
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const recentUserId = seedHistory(harness, path === "pre-prompt" ? 2500 : 650);
|
||||
|
||||
if (path === "manual") {
|
||||
await harness.session.compact();
|
||||
} else {
|
||||
harness.setResponses(
|
||||
path === "overflow"
|
||||
? [
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "prompt is too long" }),
|
||||
fauxAssistantMessage("recovered"),
|
||||
]
|
||||
: [fauxAssistantMessage(path === "post-run" ? "z".repeat(8000) : "done")],
|
||||
);
|
||||
await harness.session.prompt("continue");
|
||||
}
|
||||
|
||||
expect(preparations).toHaveLength(1);
|
||||
expect(preparations[0]?.preparation.settings).toEqual({
|
||||
enabled: path !== "manual",
|
||||
reserveTokens: 2000,
|
||||
keepRecentTokens: 150,
|
||||
});
|
||||
expect(preparations[0]?.reason).toBe(
|
||||
path === "manual" ? "manual" : path === "overflow" ? "overflow" : "threshold",
|
||||
);
|
||||
if (path === "manual" || path === "pre-prompt") {
|
||||
expect(preparations[0]?.preparation.firstKeptEntryId).toBe(recentUserId);
|
||||
}
|
||||
expect(harness.eventsOfType("compaction_end")).toHaveLength(1);
|
||||
expect(harness.eventsOfType("compaction_end")[0]).toMatchObject({
|
||||
aborted: false,
|
||||
willRetry: path === "overflow",
|
||||
result: { summary: "compacted history" },
|
||||
});
|
||||
expect(harness.getPendingResponseCount()).toBe(0);
|
||||
},
|
||||
);
|
||||
|
||||
it.each(["manual", "automatic"] as const)("passes resolved budgets to built-in %s summarization", async (path) => {
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "faux-1", contextWindow: 4000, maxTokens: 3000 }],
|
||||
tools: [],
|
||||
settings: {
|
||||
compaction: {
|
||||
reserveTokens: 10,
|
||||
modelOverrides: { "faux/faux-1": { reserveTokens: 2000, keepRecentTokens: 150 } },
|
||||
},
|
||||
},
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const recentUserId = seedHistory(harness, 2500);
|
||||
const budgets: Array<number | undefined> = [];
|
||||
harness.setResponses([
|
||||
(_context, options) => {
|
||||
budgets.push(options?.maxTokens);
|
||||
return fauxAssistantMessage("built-in summary");
|
||||
},
|
||||
...(path === "automatic" ? [fauxAssistantMessage("done")] : []),
|
||||
]);
|
||||
if (path === "manual") await harness.session.compact();
|
||||
else await harness.session.prompt("continue");
|
||||
expect(budgets).toEqual([1600]);
|
||||
expect(harness.sessionManager.getEntries().find((entry) => entry.type === "compaction")).toMatchObject({
|
||||
firstKeptEntryId: recentUserId,
|
||||
summary: "built-in summary",
|
||||
});
|
||||
expect(harness.getPendingResponseCount()).toBe(0);
|
||||
});
|
||||
|
||||
it("uses the newly selected model without changing ordinary settings", async () => {
|
||||
const harness = await createHarness({
|
||||
models: [
|
||||
{ id: "small", contextWindow: 4000 },
|
||||
{ id: "big", contextWindow: 10000 },
|
||||
],
|
||||
tools: [],
|
||||
settings: {
|
||||
compaction: {
|
||||
reserveTokens: 10,
|
||||
modelOverrides: { "faux/big": { reserveTokens: 8000, keepRecentTokens: 150 } },
|
||||
},
|
||||
},
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", (event) => ({
|
||||
compaction: {
|
||||
summary: "big model summary",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
},
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
seedHistory(harness, 2500);
|
||||
harness.setResponses([fauxAssistantMessage("small response"), fauxAssistantMessage("big response")]);
|
||||
await harness.session.prompt("continue on small");
|
||||
expect(harness.eventsOfType("compaction_start")).toHaveLength(0);
|
||||
// Retain usage from the small model: the next check must use the active big model's policy.
|
||||
seedHistory(harness, 2500);
|
||||
await harness.session.setModel(harness.getModel("big")!);
|
||||
await harness.session.prompt("continue on big");
|
||||
expect(harness.eventsOfType("compaction_end")).toHaveLength(1);
|
||||
expect(harness.eventsOfType("compaction_end")[0]?.result?.summary).toBe("big model summary");
|
||||
expect(harness.settingsManager.getCompactionReserveTokens()).toBe(10);
|
||||
await harness.session.setModel(harness.getModel("small")!);
|
||||
expect(harness.settingsManager.getCompactionReserveTokens(harness.session.model)).toBe(10);
|
||||
});
|
||||
|
||||
it("captures model identity before awaiting summarization auth", async () => {
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "first" }, { id: "second" }],
|
||||
settings: {
|
||||
compaction: {
|
||||
modelOverrides: {
|
||||
"faux/first": { reserveTokens: 2000, keepRecentTokens: 150 },
|
||||
"faux/second": { reserveTokens: 4000, keepRecentTokens: 20000 },
|
||||
},
|
||||
},
|
||||
},
|
||||
});
|
||||
harnesses.push(harness);
|
||||
seedHistory(harness);
|
||||
const getAuth = harness.session.modelRuntime.getAuth.bind(harness.session.modelRuntime);
|
||||
vi.spyOn(harness.session.modelRuntime, "getAuth").mockImplementation(async (model, ...args) => {
|
||||
harness.session.agent.state.model = harness.getModel("second")!;
|
||||
return getAuth(model, ...args);
|
||||
});
|
||||
const requests: Array<{ id: string; maxTokens: number | undefined }> = [];
|
||||
harness.setResponses([
|
||||
(_context, options, _state, model) => {
|
||||
requests.push({ id: model.id, maxTokens: options?.maxTokens });
|
||||
return fauxAssistantMessage("summary");
|
||||
},
|
||||
]);
|
||||
await harness.session.compact();
|
||||
expect(requests).toEqual([{ id: "first", maxTokens: 1600 }]);
|
||||
});
|
||||
});
|
||||
@@ -1,12 +1,14 @@
|
||||
import type { AgentMessage, AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import {
|
||||
type AssistantMessage,
|
||||
type Context,
|
||||
createAssistantMessageEventStream,
|
||||
fauxAssistantMessage,
|
||||
fauxToolCall,
|
||||
getCurrentSystemPrompt,
|
||||
getCurrentTools,
|
||||
type Model,
|
||||
type SimpleStreamOptions,
|
||||
type TranscriptContext,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
@@ -55,7 +57,7 @@ function createAssistant(
|
||||
function useSummaryStreamFn(
|
||||
harness: Harness,
|
||||
summary: string,
|
||||
onRequest?: (context: Context, options: SimpleStreamOptions | undefined) => void,
|
||||
onRequest?: (context: TranscriptContext, options: SimpleStreamOptions | undefined) => void,
|
||||
): () => number {
|
||||
let callCount = 0;
|
||||
harness.session.agent.streamFunction = (model, context, options) => {
|
||||
@@ -180,7 +182,57 @@ describe("AgentSession compaction characterization", () => {
|
||||
expect(statsAfter.tokens.cacheRead).toBe(statsBefore.tokens.cacheRead + summaryUsage.cacheRead);
|
||||
expect(statsAfter.tokens.cacheWrite).toBe(statsBefore.tokens.cacheWrite + summaryUsage.cacheWrite);
|
||||
expect(statsAfter.cost).toBe(statsBefore.cost + summaryUsage.cost.total);
|
||||
expect(harness.session.messages[0]?.role).toBe("compactionSummary");
|
||||
expect(harness.session.messages[0]?.role).toBe("system");
|
||||
expect(harness.session.messages[1]?.role).toBe("compactionSummary");
|
||||
});
|
||||
|
||||
it("checkpoints the replayed system state and folds summarized and retained system patches into it", async () => {
|
||||
const harness = await createHarness();
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([fauxAssistantMessage("declared")]);
|
||||
await harness.session.prompt("declare the prompt");
|
||||
const declared = harness.session.messages[0];
|
||||
if (declared?.role !== "system") throw new Error("expected declared system message");
|
||||
|
||||
harness.sessionManager.appendMessage({
|
||||
role: "system",
|
||||
content: "summarized instruction",
|
||||
sections: { early: "<early>1</early>" },
|
||||
toolsRemoved: [{ name: "bash" }],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
const firstKeptEntryId = harness.sessionManager.appendMessage({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "kept before patch" }],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
harness.sessionManager.appendMessage({
|
||||
role: "system",
|
||||
content: "retained instruction",
|
||||
sections: { extra: "<extra>late</extra>" },
|
||||
toolsRemoved: [{ name: "read" }],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
harness.sessionManager.appendMessage({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "kept after patch" }],
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
harness.sessionManager.appendCompaction("compacted", firstKeptEntryId, 100);
|
||||
|
||||
const messages = harness.sessionManager.buildSessionContext().messages;
|
||||
expect(messages.map((message) => message.role)).toEqual(["system", "compactionSummary", "user", "user"]);
|
||||
const checkpoint = messages[0];
|
||||
if (checkpoint?.role !== "system") throw new Error("expected checkpoint system message");
|
||||
expect(checkpoint.content).toBe("summarized instruction\n\nretained instruction");
|
||||
expect(checkpoint.sections).toEqual({
|
||||
...declared.sections,
|
||||
early: "<early>1</early>",
|
||||
extra: "<extra>late</extra>",
|
||||
});
|
||||
expect(checkpoint.toolsAdded?.map((tool) => tool.name)).toEqual(
|
||||
harness.session.getActiveToolNames().filter((name) => name !== "read" && name !== "bash"),
|
||||
);
|
||||
});
|
||||
|
||||
it("allows a queued prompt to start when manual compaction ends", async () => {
|
||||
@@ -230,6 +282,8 @@ describe("AgentSession compaction characterization", () => {
|
||||
it("throws when compacting without configured auth", async () => {
|
||||
const harness = await createHarness({ withConfiguredAuth: false });
|
||||
harnesses.push(harness);
|
||||
// Auth is resolved only when Pi summarizes itself, after checking there is something to compact.
|
||||
seedCompactableSession(harness);
|
||||
|
||||
await expect(harness.session.compact()).rejects.toThrow(`No API key found for ${harness.getModel().provider}.`);
|
||||
});
|
||||
@@ -267,13 +321,12 @@ describe("AgentSession compaction characterization", () => {
|
||||
streamSimple: () => createAssistantMessageEventStream(),
|
||||
});
|
||||
seedCompactableSession(harness);
|
||||
harness.setResponses([
|
||||
(_context, options) => {
|
||||
expect(options?.apiKey).toBeUndefined();
|
||||
expect(options?.headers).toEqual({ Authorization: "Bearer ambient-token" });
|
||||
return fauxAssistantMessage("summary with bearer auth");
|
||||
},
|
||||
]);
|
||||
const summaryResponse = (_context: TranscriptContext, options: SimpleStreamOptions | undefined) => {
|
||||
expect(options?.apiKey).toBeUndefined();
|
||||
expect(options?.headers).toEqual({ Authorization: "Bearer ambient-token" });
|
||||
return fauxAssistantMessage("summary with bearer auth");
|
||||
};
|
||||
harness.setResponses([summaryResponse]);
|
||||
|
||||
const result = await harness.session.compact();
|
||||
|
||||
@@ -291,7 +344,7 @@ describe("AgentSession compaction characterization", () => {
|
||||
harness.session.agent.sessionId = "active-routing-session";
|
||||
harness.session.agent.transport = "websocket";
|
||||
|
||||
let requestContext: Context | undefined;
|
||||
let requestContext: TranscriptContext | undefined;
|
||||
let requestOptions: SimpleStreamOptions | undefined;
|
||||
useSummaryStreamFn(harness, "standalone summary", (context, options) => {
|
||||
requestContext = context;
|
||||
@@ -301,9 +354,10 @@ describe("AgentSession compaction characterization", () => {
|
||||
await harness.session.compact();
|
||||
|
||||
expect(transformContext).not.toHaveBeenCalled();
|
||||
expect(requestContext?.systemPrompt).not.toBe(harness.session.agent.state.systemPrompt);
|
||||
expect(requestContext?.tools).toBeUndefined();
|
||||
expect(JSON.stringify(requestContext?.messages)).toContain("<conversation>");
|
||||
expect(getCurrentSystemPrompt(requestContext?.messages ?? [])).not.toBe(harness.session.agent.state.systemPrompt);
|
||||
expect(getCurrentTools(requestContext?.messages ?? [])).toEqual([]);
|
||||
// Regression test for #9652: split-turn summaries use a clear Markdown conversation boundary.
|
||||
expect(JSON.stringify(requestContext?.messages)).toContain("# Conversation\\n[User]: message to compact");
|
||||
expect(requestOptions).toMatchObject({ cacheRetention: "none" });
|
||||
expect(requestOptions?.sessionId).not.toBe("active-routing-session");
|
||||
expect(requestOptions?.transport).toBeUndefined();
|
||||
@@ -419,64 +473,81 @@ describe("AgentSession compaction characterization", () => {
|
||||
expect(harness.session.getLastAssistantText()).toBe("completed response");
|
||||
});
|
||||
|
||||
it("compacts after a tool result before the next assistant request in the same run", async () => {
|
||||
const toolResult = `large-tool-result:${"x".repeat(6800)}`;
|
||||
const largeTool: AgentTool = {
|
||||
name: "large_result",
|
||||
label: "Large result",
|
||||
description: "Returns enough content to cross the compaction threshold",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({ content: [{ type: "text", text: toolResult }], details: {} }),
|
||||
};
|
||||
const order: string[] = [];
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "faux-1", contextWindow: 2600, maxTokens: 100 }],
|
||||
settings: { compaction: { enabled: true, reserveTokens: 400, keepRecentTokens: 1750 } },
|
||||
tools: [largeTool],
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", (event) => {
|
||||
order.push("compaction");
|
||||
return {
|
||||
compaction: {
|
||||
summary: "compacted history",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
details: {},
|
||||
},
|
||||
};
|
||||
});
|
||||
// Regression coverage for #8133: model overrides must also apply between assistant turns.
|
||||
// Regression coverage for #9740: an oversized trailing tool result must still produce a cut point.
|
||||
it.each([false, true])(
|
||||
"compacts after an oversized tool result in the same run (model override: %s)",
|
||||
async (modelOverride) => {
|
||||
const toolResult = `large-tool-result:${"x".repeat(8000)}`;
|
||||
const largeTool: AgentTool = {
|
||||
name: "large_result",
|
||||
label: "Large result",
|
||||
description: "Returns enough content to cross the compaction threshold",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({ content: [{ type: "text", text: toolResult }], details: {} }),
|
||||
};
|
||||
const order: string[] = [];
|
||||
const observedSettings: unknown[] = [];
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "faux-1", contextWindow: 2600, maxTokens: 100 }],
|
||||
settings: {
|
||||
compaction: modelOverride
|
||||
? {
|
||||
enabled: true,
|
||||
reserveTokens: 0,
|
||||
keepRecentTokens: 20000,
|
||||
modelOverrides: { "faux/faux-1": { reserveTokens: 400, keepRecentTokens: 1750 } },
|
||||
}
|
||||
: { enabled: true, reserveTokens: 400, keepRecentTokens: 1750 },
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
let resumedRequest = "";
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(`old-history:${"a".repeat(800)}`),
|
||||
fauxAssistantMessage(`recent-history:${"b".repeat(800)}`),
|
||||
fauxAssistantMessage(fauxToolCall("large_result", {}), { stopReason: "toolUse" }),
|
||||
(context) => {
|
||||
order.push("provider");
|
||||
resumedRequest = JSON.stringify(context.messages);
|
||||
return fauxAssistantMessage("finished after compaction");
|
||||
},
|
||||
]);
|
||||
tools: [largeTool],
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", (event) => {
|
||||
order.push("compaction");
|
||||
observedSettings.push(event.preparation.settings);
|
||||
return {
|
||||
compaction: {
|
||||
summary: "compacted history",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
details: {},
|
||||
},
|
||||
};
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
let resumedRequest = "";
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(`old-history:${"a".repeat(800)}`),
|
||||
fauxAssistantMessage(`recent-history:${"b".repeat(800)}`),
|
||||
fauxAssistantMessage(fauxToolCall("large_result", {}), { stopReason: "toolUse" }),
|
||||
(context) => {
|
||||
order.push("provider");
|
||||
resumedRequest = JSON.stringify(context.messages);
|
||||
return fauxAssistantMessage("finished after compaction");
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("seed old history");
|
||||
await harness.session.prompt("seed recent history");
|
||||
const agentStartsBefore = harness.eventsOfType("agent_start").length;
|
||||
await harness.session.prompt("run the large tool");
|
||||
await harness.session.prompt("seed old history");
|
||||
await harness.session.prompt("seed recent history");
|
||||
const agentStartsBefore = harness.eventsOfType("agent_start").length;
|
||||
await harness.session.prompt("run the large tool");
|
||||
|
||||
expect(order).toEqual(["compaction", "provider"]);
|
||||
expect(harness.eventsOfType("agent_start")).toHaveLength(agentStartsBefore + 1);
|
||||
expect(harness.eventsOfType("compaction_start").at(-1)).toEqual({
|
||||
type: "compaction_start",
|
||||
reason: "threshold",
|
||||
});
|
||||
expect(resumedRequest).toContain("compacted history");
|
||||
expect(resumedRequest).toContain("large-tool-result");
|
||||
expect(harness.session.getLastAssistantText()).toBe("finished after compaction");
|
||||
});
|
||||
expect(order.slice(0, 2)).toEqual(["compaction", "provider"]);
|
||||
expect(observedSettings[0]).toEqual({ enabled: true, reserveTokens: 400, keepRecentTokens: 1750 });
|
||||
expect(harness.eventsOfType("agent_start")).toHaveLength(agentStartsBefore + 1);
|
||||
expect(harness.eventsOfType("compaction_start").at(-1)).toEqual({
|
||||
type: "compaction_start",
|
||||
reason: "threshold",
|
||||
});
|
||||
expect(resumedRequest).toContain("compacted history");
|
||||
expect(resumedRequest).toContain("large-tool-result");
|
||||
expect(harness.session.getLastAssistantText()).toBe("finished after compaction");
|
||||
},
|
||||
);
|
||||
|
||||
it("includes steering queued during compaction in the resumed assistant request", async () => {
|
||||
const largeTool: AgentTool = {
|
||||
|
||||
273
third_party/pi/packages/coding-agent/test/suite/agent-session-mcp-oauth.test.ts
vendored
Normal file
273
third_party/pi/packages/coding-agent/test/suite/agent-session-mcp-oauth.test.ts
vendored
Normal file
@@ -0,0 +1,273 @@
|
||||
import { mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { type AddressInfo, createServer } from "node:net";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
|
||||
import type { ToolResultMessage } from "@earendil-works/pi-ai/compat";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { InMemoryAuthStorageBackend } from "../../src/core/auth-storage.ts";
|
||||
import { emitSessionShutdownEvent } from "../../src/core/extensions/runner.ts";
|
||||
import { runMcpCommand } from "../../src/extensions/mcp/cli.ts";
|
||||
import type { McpOAuthConfig, McpServerEntry } from "../../src/extensions/mcp/config.ts";
|
||||
import { createMcpExtension } from "../../src/extensions/mcp/index.ts";
|
||||
import { McpOAuthCredentialStore } from "../../src/extensions/mcp/oauth.ts";
|
||||
import { createHarness, createTestUiContext, getMessageText, type Harness } from "./harness.ts";
|
||||
import { startOAuthMcpServer } from "./mcp-oauth-server.ts";
|
||||
|
||||
describe("AgentSession MCP OAuth", () => {
|
||||
const cleanups: (() => Promise<void> | void)[] = [];
|
||||
|
||||
afterEach(async () => {
|
||||
while (cleanups.length > 0) await cleanups.pop()?.();
|
||||
});
|
||||
|
||||
async function setup(browser: "follow" | "paste", oauth?: McpOAuthConfig) {
|
||||
const server = await startOAuthMcpServer();
|
||||
cleanups.push(server.close);
|
||||
const backend = new InMemoryAuthStorageBackend();
|
||||
const entry: McpServerEntry = {
|
||||
name: "issues",
|
||||
config: { url: server.url, exposure: "direct", ...(oauth ? { oauth } : {}) },
|
||||
source: "test",
|
||||
};
|
||||
const notifications: string[] = [];
|
||||
const opened: URL[] = [];
|
||||
let redirectLocation: Promise<string> | undefined;
|
||||
const harness: Harness = await createHarness({
|
||||
initialActiveToolNames: [],
|
||||
extensionFactories: [
|
||||
createMcpExtension({
|
||||
loadConfig: () => ({ servers: [entry], errors: [] }),
|
||||
credentials: new McpOAuthCredentialStore(backend),
|
||||
openUrl: (url) => {
|
||||
opened.push(new URL(url));
|
||||
if (browser === "follow") {
|
||||
// The browser follows the authorization redirect to the loopback callback.
|
||||
void fetch(url);
|
||||
} else {
|
||||
// The browser cannot reach the callback; the user pastes the redirect URL.
|
||||
redirectLocation = fetch(url, { redirect: "manual" }).then(
|
||||
(response) => response.headers.get("location") ?? "",
|
||||
);
|
||||
}
|
||||
},
|
||||
}),
|
||||
],
|
||||
});
|
||||
cleanups.push(() => harness.cleanup());
|
||||
await harness.session.bindExtensions({
|
||||
uiContext: createTestUiContext({
|
||||
notify: (message) => notifications.push(message),
|
||||
// The paste prompt waits until sign-in completes unless the user pastes the redirect URL.
|
||||
input: (_title, _placeholder, opts) =>
|
||||
browser === "paste"
|
||||
? Promise.resolve(redirectLocation)
|
||||
: new Promise((resolve) =>
|
||||
opts?.signal?.addEventListener("abort", () => resolve(undefined), { once: true }),
|
||||
),
|
||||
}),
|
||||
});
|
||||
return { harness, server, notifications, backend, opened };
|
||||
}
|
||||
|
||||
async function callWhoami(harness: Harness): Promise<ToolResultMessage> {
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage([fauxToolCall("mcp__issues__whoami", {})], { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
const before = harness.session.messages.length;
|
||||
await harness.session.prompt("who am i");
|
||||
const result = harness.session.messages
|
||||
.slice(before)
|
||||
.find((message): message is ToolResultMessage => message.role === "toolResult");
|
||||
if (!result) throw new Error("no tool result");
|
||||
return result;
|
||||
}
|
||||
|
||||
it("signs in through the browser, refreshes expired tokens, and signs out", async () => {
|
||||
const { harness, server, notifications, backend } = await setup("follow");
|
||||
|
||||
await harness.session.prompt("/mcp");
|
||||
// Startup problems are reported once, pointing to /mcp.
|
||||
expect(notifications).toContain("MCP servers need attention:\n issues: needs sign-in\nRun /mcp to fix.");
|
||||
expect(notifications.at(-1)).toBe("issues: needs sign-in, run /mcp login issues (direct)");
|
||||
|
||||
await harness.session.prompt("/mcp login issues");
|
||||
expect(notifications.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
expect(server.log).toEqual(["401 none", "register", "token code"]);
|
||||
expect(backend.withLock((current) => ({ result: current }))).toContain('"access_token": "access-1"');
|
||||
|
||||
expect(getMessageText(await callWhoami(harness))).toBe("token access-1");
|
||||
|
||||
// An expired access token is refreshed without user interaction.
|
||||
server.expireAccessTokens();
|
||||
expect(getMessageText(await callWhoami(harness))).toBe("token access-2");
|
||||
expect(server.log.slice(-3)).toEqual(["401 access-1", "token refresh", "call access-2"]);
|
||||
|
||||
// A token past its expiry is refreshed before the request, without a 401 round trip.
|
||||
backend.withLock((current) => {
|
||||
const states = JSON.parse(current ?? "{}") as Record<string, { tokensExpireAt?: number }>;
|
||||
for (const state of Object.values(states)) state.tokensExpireAt = Date.now() - 1_000;
|
||||
return { result: undefined, next: JSON.stringify(states) };
|
||||
});
|
||||
expect(getMessageText(await callWhoami(harness))).toBe("token access-3");
|
||||
expect(server.log.slice(-2)).toEqual(["token refresh", "call access-3"]);
|
||||
|
||||
await harness.session.prompt("/mcp logout issues");
|
||||
expect(notifications.at(-1)).toBe('Signed out of MCP server "issues".');
|
||||
const result = await callWhoami(harness);
|
||||
expect(result.isError).toBe(true);
|
||||
expect(getMessageText(result)).toBe('MCP server "issues" requires sign-in. Run /mcp to sign in.');
|
||||
});
|
||||
|
||||
it("accepts a pasted redirect URL when the browser cannot reach the callback", async () => {
|
||||
const { harness, server, notifications } = await setup("paste");
|
||||
|
||||
await harness.session.prompt("/mcp login");
|
||||
expect(notifications.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
expect(getMessageText(await callWhoami(harness))).toBe(`token access-1`);
|
||||
expect(server.log).toContain("token code");
|
||||
});
|
||||
|
||||
/** Emit session_shutdown like quitting does; resolves to how long the handlers took. */
|
||||
async function shutdown(harness: Harness): Promise<number> {
|
||||
const start = Date.now();
|
||||
await emitSessionShutdownEvent(harness.session.extensionRunner, { type: "session_shutdown", reason: "quit" });
|
||||
return Date.now() - start;
|
||||
}
|
||||
|
||||
// #10565
|
||||
for (const path of ["/.well-known/oauth-authorization-server", "/token"]) {
|
||||
it(`cancels a sign-in waiting on ${path} when the session shuts down`, async () => {
|
||||
const { harness, server, notifications } = await setup("follow");
|
||||
server.stall.add(path);
|
||||
const login = harness.session.prompt("/mcp login issues");
|
||||
await vi.waitFor(() => expect(server.stalled).toHaveLength(1), { timeout: 5_000 });
|
||||
|
||||
// Shutdown waits for the sign-in to clean up, which aborting makes immediate.
|
||||
expect(await shutdown(harness)).toBeLessThan(2_000);
|
||||
await login;
|
||||
// The request is aborted instead of left open, and the ended session reports nothing.
|
||||
await vi.waitFor(() => expect(server.stalled[0]?.request.socket.destroyed).toBe(true));
|
||||
expect(notifications.filter((message) => message.startsWith("Sign-in"))).toEqual([]);
|
||||
});
|
||||
}
|
||||
|
||||
// #10565
|
||||
it("closes the session without refreshing an expiring token", async () => {
|
||||
const { harness, server, notifications, backend } = await setup("follow");
|
||||
await harness.session.prompt("/mcp login issues");
|
||||
expect(notifications.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
// Still accepted, but close enough to expiry that the next request would refresh it first.
|
||||
backend.withLock((current) => {
|
||||
const states = JSON.parse(current ?? "{}") as Record<string, { tokensExpireAt?: number }>;
|
||||
for (const state of Object.values(states)) state.tokensExpireAt = Date.now() + 10_000;
|
||||
return { result: undefined, next: JSON.stringify(states) };
|
||||
});
|
||||
server.stall.add("/token");
|
||||
|
||||
expect(await shutdown(harness)).toBeLessThan(2_000);
|
||||
expect(server.stalled).toEqual([]);
|
||||
expect(server.deletes).toEqual(["access-1"]);
|
||||
});
|
||||
|
||||
async function freePort(): Promise<number> {
|
||||
return new Promise<number>((resolve) => {
|
||||
const probe = createServer().listen(0, "127.0.0.1", () => {
|
||||
const address = probe.address() as AddressInfo;
|
||||
probe.close(() => resolve(address.port));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
it("uses the configured callback URL and scope", async () => {
|
||||
const callbackUrl = `http://localhost:${await freePort()}/callback`;
|
||||
const { harness, notifications, opened } = await setup("follow", { callbackUrl, scope: "issues:read" });
|
||||
|
||||
await harness.session.prompt("/mcp login issues");
|
||||
expect(notifications.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
expect(opened[0].searchParams.get("redirect_uri")).toBe(callbackUrl);
|
||||
expect(opened[0].searchParams.get("scope")).toBe("issues:read");
|
||||
expect(getMessageText(await callWhoami(harness))).toBe("token access-1");
|
||||
});
|
||||
|
||||
// #10226
|
||||
it("registers with the configured client name", async () => {
|
||||
const { harness, server, notifications } = await setup("follow", { clientName: "Claude Code" });
|
||||
await harness.session.prompt("/mcp login issues");
|
||||
expect(notifications.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
expect(server.registrations.map((metadata) => metadata.client_name)).toEqual(["Claude Code"]);
|
||||
|
||||
const fallback = await setup("follow");
|
||||
await fallback.harness.session.prompt("/mcp login issues");
|
||||
expect(fallback.server.registrations.map((metadata) => metadata.client_name)).toEqual(["pi"]);
|
||||
// OpenID Connect servers reject the loopback redirect URI of a `web` client (#10493).
|
||||
expect(fallback.server.registrations.map((metadata) => metadata.application_type)).toEqual(["native"]);
|
||||
});
|
||||
|
||||
it("adds the listening port to a callback URL without one", async () => {
|
||||
const { harness, notifications, opened } = await setup("follow", { callbackUrl: "http://127.0.0.1/oauth/done" });
|
||||
await harness.session.prompt("/mcp login issues");
|
||||
expect(notifications.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
expect(opened[0].searchParams.get("redirect_uri")).toMatch(/^http:\/\/127\.0\.0\.1:\d+\/oauth\/done$/);
|
||||
|
||||
const port = await freePort();
|
||||
const fixed = await setup("follow", { callbackUrl: "http://127.0.0.1/oauth/done", callbackPort: port });
|
||||
await fixed.harness.session.prompt("/mcp login issues");
|
||||
expect(fixed.opened[0].searchParams.get("redirect_uri")).toBe(`http://127.0.0.1:${port}/oauth/done`);
|
||||
});
|
||||
|
||||
it("gives up on pi mcp login after --timeout, also while the authorization server hangs", async () => {
|
||||
const server = await startOAuthMcpServer();
|
||||
cleanups.push(server.close);
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "pi-mcp-login-"));
|
||||
cleanups.push(() => rmSync(agentDir, { recursive: true, force: true }));
|
||||
writeFileSync(join(agentDir, "mcp.json"), JSON.stringify({ mcpServers: { issues: { url: server.url } } }));
|
||||
const login = async () => {
|
||||
const output: string[] = [];
|
||||
const exitCode = await runMcpCommand(["login", "issues", "--timeout", "0.5"], {
|
||||
cwd: agentDir,
|
||||
agentDir,
|
||||
credentials: new McpOAuthCredentialStore(new InMemoryAuthStorageBackend()),
|
||||
// The user never approves in the browser.
|
||||
openUrl: () => {},
|
||||
log: (line) => output.push(line),
|
||||
error: (line) => output.push(line),
|
||||
});
|
||||
return { exitCode, output };
|
||||
};
|
||||
|
||||
const unapproved = await login();
|
||||
expect(unapproved.exitCode).toBe(1);
|
||||
expect(unapproved.output.at(-1)).toContain("was cancelled or not completed within");
|
||||
|
||||
// #10565
|
||||
server.stall.add("/.well-known/oauth-authorization-server");
|
||||
const stalled = await login();
|
||||
expect(stalled.exitCode).toBe(1);
|
||||
expect(stalled.output.at(-1)).toContain("was cancelled or not completed within");
|
||||
});
|
||||
|
||||
it("uses credentials from pi mcp login on the next turn", async () => {
|
||||
const { harness, server, backend } = await setup("follow");
|
||||
const agentDir = mkdtempSync(join(tmpdir(), "pi-mcp-login-"));
|
||||
cleanups.push(() => rmSync(agentDir, { recursive: true, force: true }));
|
||||
writeFileSync(join(agentDir, "mcp.json"), JSON.stringify({ mcpServers: { issues: { url: server.url } } }));
|
||||
|
||||
// The agent runs `pi mcp login issues` through bash; the user approves in the browser.
|
||||
const output: string[] = [];
|
||||
const exitCode = await runMcpCommand(["login", "issues"], {
|
||||
cwd: agentDir,
|
||||
agentDir,
|
||||
credentials: new McpOAuthCredentialStore(backend),
|
||||
openUrl: (url) => void fetch(url),
|
||||
log: (line) => output.push(line),
|
||||
error: (line) => output.push(line),
|
||||
});
|
||||
expect(exitCode).toBe(0);
|
||||
expect(output.at(-1)).toBe('Signed in to MCP server "issues" (1 tools).');
|
||||
|
||||
// The session still waits for a sign-in, and reconnects when the next turn starts.
|
||||
expect(getMessageText(await callWhoami(harness))).toBe("token access-1");
|
||||
});
|
||||
});
|
||||
1273
third_party/pi/packages/coding-agent/test/suite/agent-session-mcp.test.ts
vendored
Normal file
1273
third_party/pi/packages/coding-agent/test/suite/agent-session-mcp.test.ts
vendored
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,5 +1,12 @@
|
||||
import type { AgentTool, ThinkingLevel } from "@earendil-works/pi-agent-core";
|
||||
import { fauxAssistantMessage, fauxToolCall, type Model, type Usage } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
fauxAssistantMessage,
|
||||
fauxToolCall,
|
||||
getCurrentSystemPrompt,
|
||||
type JsonObject,
|
||||
type Model,
|
||||
type Usage,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { BuildSystemPromptOptions, ExtensionAPI } from "../../src/index.ts";
|
||||
@@ -339,7 +346,12 @@ describe("AgentSession model and extension characterization", () => {
|
||||
|
||||
expect(getAssistantTexts(harness)).toContain("patched result");
|
||||
const toolResult = harness.session.messages.find(
|
||||
(message) => message.role === "toolResult" && message.details?.patched === true,
|
||||
(message) =>
|
||||
message.role === "toolResult" &&
|
||||
typeof message.details === "object" &&
|
||||
message.details !== null &&
|
||||
!Array.isArray(message.details) &&
|
||||
(message.details as JsonObject).patched === true,
|
||||
);
|
||||
expect(observedToolUsage).toEqual(toolUsage);
|
||||
expect(toolResult).toBeDefined();
|
||||
@@ -474,7 +486,7 @@ describe("AgentSession model and extension characterization", () => {
|
||||
let sawInjectedUserMessage = false;
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
providerSystemPrompt = context.systemPrompt ?? "";
|
||||
providerSystemPrompt = getCurrentSystemPrompt(context.messages);
|
||||
sawInjectedUserMessage = context.messages.some(
|
||||
(message) =>
|
||||
message.role === "user" &&
|
||||
|
||||
@@ -2,20 +2,40 @@ import { mkdirSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import { fauxAssistantMessage, fauxToolCall, type Model } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
fauxAssistantMessage,
|
||||
fauxToolCall,
|
||||
getCurrentSystemPrompt,
|
||||
type Model,
|
||||
type TranscriptContext,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { ExtensionAPI, InputEvent } from "../../src/core/extensions/index.ts";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { BeforeAgentStartEvent, ExtensionAPI, InputEvent } from "../../src/core/extensions/index.ts";
|
||||
import type { PromptTemplate } from "../../src/core/prompt-templates.ts";
|
||||
import { createSyntheticSourceInfo } from "../../src/core/source-info.ts";
|
||||
import { createTestResourceLoader } from "../utilities.ts";
|
||||
import { createHarness, getMessageText, type Harness } from "./harness.ts";
|
||||
|
||||
const processImage = vi.hoisted(() =>
|
||||
vi.fn(async (_bytes: Uint8Array, mimeType: string) => ({
|
||||
ok: true as const,
|
||||
data: Buffer.from("normalized").toString("base64"),
|
||||
mimeType,
|
||||
hints: [],
|
||||
})),
|
||||
);
|
||||
vi.mock("../../src/utils/image-process.ts", () => ({ processImage }));
|
||||
|
||||
const TINY_PNG_BASE64 =
|
||||
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8DwHwAFBQIAX8jx0gAAAABJRU5ErkJggg==";
|
||||
|
||||
describe("AgentSession prompt characterization", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
const tempDirs: string[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
processImage.mockClear();
|
||||
while (harnesses.length > 0) {
|
||||
harnesses.pop()?.cleanup();
|
||||
}
|
||||
@@ -35,8 +55,8 @@ describe("AgentSession prompt characterization", () => {
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["user", "assistant"]);
|
||||
expect(getMessageText(harness.session.messages[0]!)).toBe("hi");
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["system", "user", "assistant"]);
|
||||
expect(getMessageText(harness.session.messages[1]!)).toBe("hi");
|
||||
expect(harness.getPendingResponseCount()).toBe(0);
|
||||
});
|
||||
|
||||
@@ -68,13 +88,14 @@ describe("AgentSession prompt characterization", () => {
|
||||
|
||||
expect(toolRuns).toEqual(["hello"]);
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual([
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"assistant",
|
||||
]);
|
||||
expect(harness.session.messages[2]?.role).toBe("toolResult");
|
||||
expect(harness.session.messages[3]?.role).toBe("assistant");
|
||||
expect(harness.session.messages[3]?.role).toBe("toolResult");
|
||||
expect(harness.session.messages[4]?.role).toBe("assistant");
|
||||
});
|
||||
|
||||
it("executes multiple tool calls from one response and continues with a single follow-up response", async () => {
|
||||
@@ -144,6 +165,44 @@ describe("AgentSession prompt characterization", () => {
|
||||
expect(sawImage).toBe(true);
|
||||
});
|
||||
|
||||
// Regression test for https://github.com/earendil-works/pi/issues/9631
|
||||
it("uses the model selected by before_agent_start for image normalization", async () => {
|
||||
let strictModel: Model<string> | undefined;
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "wide" }, { id: "strict" }],
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", async () => {
|
||||
if (!strictModel) throw new Error("Expected strict model");
|
||||
await pi.setModel(strictModel);
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
strictModel = harness.getModel("strict");
|
||||
if (!strictModel) throw new Error("Expected strict model");
|
||||
const resizeOptions = { maxWidth: 1000, maxHeight: 1000, maxBytes: 500000, jpegQuality: 70 };
|
||||
strictModel.inputLimits = { images: { resize: resizeOptions } };
|
||||
harness.setResponses([fauxAssistantMessage("done")]);
|
||||
|
||||
await harness.session.prompt("inspect", {
|
||||
images: [{ type: "image", data: TINY_PNG_BASE64, mimeType: "image/png" }],
|
||||
});
|
||||
|
||||
expect(harness.session.model?.id).toBe("strict");
|
||||
expect(processImage).toHaveBeenCalledWith(expect.any(Uint8Array), "image/png", {
|
||||
autoResizeImages: true,
|
||||
resizeOptions,
|
||||
});
|
||||
const userMessage = harness.session.messages.find((message) => message.role === "user");
|
||||
expect(userMessage?.content).toContainEqual({
|
||||
type: "image",
|
||||
data: Buffer.from("normalized").toString("base64"),
|
||||
mimeType: "image/png",
|
||||
});
|
||||
});
|
||||
|
||||
it("expands skill commands before sending the prompt", async () => {
|
||||
const tempDir = join(tmpdir(), `pi-skill-${Date.now()}-${Math.random().toString(36).slice(2)}`);
|
||||
mkdirSync(tempDir, { recursive: true });
|
||||
@@ -318,8 +377,8 @@ describe("AgentSession prompt characterization", () => {
|
||||
|
||||
await harness.session.sendUserMessage("from extension");
|
||||
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["user", "assistant"]);
|
||||
expect(getMessageText(harness.session.messages[0]!)).toBe("from extension");
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["system", "user", "assistant"]);
|
||||
expect(getMessageText(harness.session.messages[1]!)).toBe("from extension");
|
||||
});
|
||||
|
||||
it("does not report streamingBehavior to input handlers while idle", async () => {
|
||||
@@ -503,4 +562,162 @@ describe("AgentSession prompt characterization", () => {
|
||||
`No API key found for ${harness.getModel().provider}.`,
|
||||
);
|
||||
});
|
||||
|
||||
// Issue #10267, #5581: a run started by sendMessage(..., { triggerTurn: true }) skipped
|
||||
// before_agent_start, so its second request patched away the sections handlers had added.
|
||||
describe("runs started by a custom message", () => {
|
||||
it("keeps the same system prompt for every request of the run", async () => {
|
||||
const events: BeforeAgentStartEvent[] = [];
|
||||
const noopTool: AgentTool = {
|
||||
name: "noop",
|
||||
label: "Noop",
|
||||
description: "Does nothing",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({ content: [{ type: "text", text: "ok" }], details: {} }),
|
||||
};
|
||||
const harness = await createHarness({
|
||||
tools: [noopTool],
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", (event) => {
|
||||
events.push(event);
|
||||
event.systemPromptOptions.sections.extension_rules = "Answer in one word.";
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const prompts: string[] = [];
|
||||
const record = (response: ReturnType<typeof fauxAssistantMessage>) => (context: TranscriptContext) => {
|
||||
prompts.push(getCurrentSystemPrompt(context.messages));
|
||||
return response;
|
||||
};
|
||||
harness.setResponses([
|
||||
record(fauxAssistantMessage("hi")),
|
||||
record(fauxAssistantMessage([fauxToolCall("noop", {})], { stopReason: "toolUse" })),
|
||||
record(fauxAssistantMessage("done")),
|
||||
]);
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
await harness.session.sendCustomMessage(
|
||||
{ customType: "task-done", content: "background task finished", display: true },
|
||||
{ triggerTurn: true },
|
||||
);
|
||||
|
||||
expect(prompts).toHaveLength(3);
|
||||
expect(prompts[0]).toContain("Answer in one word.");
|
||||
expect(new Set(prompts).size).toBe(1);
|
||||
expect(events.map((event) => event.messageType)).toEqual(["user", "custom"]);
|
||||
const customEvent = events[1];
|
||||
expect(customEvent.messageType === "custom" && customEvent.message.customType).toBe("task-done");
|
||||
});
|
||||
|
||||
it("queues a custom message sent while before_agent_start runs into the starting run", async () => {
|
||||
let releaseHandler: () => void = () => {};
|
||||
const handlerStarted = Promise.withResolvers<void>();
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", async (event) => {
|
||||
if (event.messageType !== "custom" || event.message.customType !== "first") return;
|
||||
handlerStarted.resolve();
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseHandler = resolve;
|
||||
});
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const requests: TranscriptContext[] = [];
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
requests.push(context);
|
||||
return fauxAssistantMessage("handled both");
|
||||
},
|
||||
]);
|
||||
|
||||
const first = harness.session.sendCustomMessage(
|
||||
{ customType: "first", content: "task one finished", display: true },
|
||||
{ triggerTurn: true },
|
||||
);
|
||||
await handlerStarted.promise;
|
||||
expect(harness.session.isIdle).toBe(false);
|
||||
const second = harness.session.sendCustomMessage(
|
||||
{ customType: "second", content: "task two finished", display: true },
|
||||
{ triggerTurn: true },
|
||||
);
|
||||
releaseHandler();
|
||||
await Promise.all([first, second]);
|
||||
|
||||
expect(requests).toHaveLength(1);
|
||||
const texts = requests[0].messages.flatMap((message) =>
|
||||
message.role === "user" && typeof message.content !== "string"
|
||||
? message.content.flatMap((part) => (part.type === "text" ? [part.text] : []))
|
||||
: [],
|
||||
);
|
||||
expect(texts).toEqual(["task one finished", "task two finished"]);
|
||||
});
|
||||
|
||||
it("includes a custom message sent without triggerTurn while the run starts in its first request", async () => {
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", () => {
|
||||
pi.sendMessage(
|
||||
{ customType: "note", content: "context from handler", display: true },
|
||||
{ triggerTurn: false },
|
||||
);
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const requests: TranscriptContext[] = [];
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
requests.push(context);
|
||||
return fauxAssistantMessage("done");
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
expect(requests).toHaveLength(1);
|
||||
expect(JSON.stringify(requests[0].messages)).toContain("context from handler");
|
||||
});
|
||||
|
||||
it("aborts a run when abort is requested while before_agent_start runs", async () => {
|
||||
let releaseHandler: () => void = () => {};
|
||||
const handlerStarted = Promise.withResolvers<void>();
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", async () => {
|
||||
handlerStarted.resolve();
|
||||
await new Promise<void>((resolve) => {
|
||||
releaseHandler = resolve;
|
||||
});
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([fauxAssistantMessage("should not complete")]);
|
||||
|
||||
const run = harness.session.sendCustomMessage(
|
||||
{ customType: "task-done", content: "background task finished", display: true },
|
||||
{ triggerTurn: true },
|
||||
);
|
||||
await handlerStarted.promise;
|
||||
const aborted = harness.session.abort();
|
||||
releaseHandler();
|
||||
await Promise.all([run, aborted]);
|
||||
|
||||
const last = harness.session.messages.at(-1);
|
||||
expect(last?.role).toBe("assistant");
|
||||
expect(last?.role === "assistant" && last.stopReason).toBe("aborted");
|
||||
expect(harness.session.isIdle).toBe(true);
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
|
||||
import type { ExtensionAPI } from "@earendil-works/pi-coding-agent";
|
||||
import type { ExtensionAPI, InputEvent } from "@earendil-works/pi-coding-agent";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { createHarness, getAssistantTexts, getMessageText, getUserTexts, type Harness } from "./harness.ts";
|
||||
@@ -155,6 +155,54 @@ describe("AgentSession queue characterization", () => {
|
||||
expect(getAssistantTexts(harness)).toContain("follow-up response");
|
||||
});
|
||||
|
||||
// Regression test for #8718.
|
||||
it("runs direct steering and follow-up messages through input handlers", async () => {
|
||||
const inputEvents: Array<Pick<InputEvent, "text" | "source" | "streamingBehavior">> = [];
|
||||
const waiting = await createWaitingHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("input", (event) => {
|
||||
inputEvents.push({
|
||||
text: event.text,
|
||||
source: event.source,
|
||||
streamingBehavior: event.streamingBehavior,
|
||||
});
|
||||
if (event.text.startsWith("handle")) return { action: "handled" };
|
||||
return { action: "transform", text: `transformed: ${event.text}` };
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
const { harness, waitForToolStart, promptPromise, releaseToolExecution } = waiting;
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(fauxToolCall("wait", {}), { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("steered"),
|
||||
fauxAssistantMessage("followed up"),
|
||||
]);
|
||||
|
||||
await waitForToolStart;
|
||||
inputEvents.length = 0;
|
||||
try {
|
||||
await harness.session.steer("steer me", undefined, { source: "rpc" });
|
||||
await harness.session.steer("handle steer", undefined, { source: "rpc" });
|
||||
await harness.session.followUp("follow me", undefined, { source: "rpc" });
|
||||
await harness.session.followUp("handle follow", undefined, { source: "rpc" });
|
||||
|
||||
expect(inputEvents).toEqual([
|
||||
{ text: "steer me", source: "rpc", streamingBehavior: "steer" },
|
||||
{ text: "handle steer", source: "rpc", streamingBehavior: "steer" },
|
||||
{ text: "follow me", source: "rpc", streamingBehavior: "followUp" },
|
||||
{ text: "handle follow", source: "rpc", streamingBehavior: "followUp" },
|
||||
]);
|
||||
expect(harness.session.getSteeringMessages()).toEqual(["transformed: steer me"]);
|
||||
expect(harness.session.getFollowUpMessages()).toEqual(["transformed: follow me"]);
|
||||
} finally {
|
||||
releaseToolExecution();
|
||||
}
|
||||
await promptPromise;
|
||||
});
|
||||
|
||||
it("delivers multiple steering messages in order in one-at-a-time mode", async () => {
|
||||
const waiting = await createWaitingHarness();
|
||||
const { harness, waitForToolStart, promptPromise, releaseToolExecution } = waiting;
|
||||
@@ -350,7 +398,12 @@ describe("AgentSession queue characterization", () => {
|
||||
await harness.session.prompt("normal prompt");
|
||||
|
||||
expect(sawCustomMessage).toBe(true);
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual(["user", "custom", "assistant"]);
|
||||
expect(harness.session.messages.map((message) => message.role)).toEqual([
|
||||
"system",
|
||||
"user",
|
||||
"custom",
|
||||
"assistant",
|
||||
]);
|
||||
});
|
||||
|
||||
it("updates pendingMessageCount and removes queued text before message_start is emitted", async () => {
|
||||
|
||||
@@ -73,6 +73,32 @@ describe("AgentSession retry and event characterization", () => {
|
||||
expect(harness.faux.state.callCount).toBe(3);
|
||||
});
|
||||
|
||||
// Regression test for #9340.
|
||||
it("finalizes retry state when abort is requested after a retry attempt fails", async () => {
|
||||
const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 0 } } });
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" }),
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" }),
|
||||
]);
|
||||
|
||||
let errorCount = 0;
|
||||
harness.session.subscribe((event) => {
|
||||
if (event.type !== "message_end" || event.message.role !== "assistant") return;
|
||||
if (event.message.stopReason === "error" && ++errorCount === 2) void harness.session.abort();
|
||||
});
|
||||
|
||||
await harness.session.prompt("test");
|
||||
|
||||
expect(harness.session.retryAttempt).toBe(0);
|
||||
expect(harness.eventsOfType("agent_end").at(-1)?.willRetry).toBe(false);
|
||||
expect(harness.eventsOfType("auto_retry_end").at(-1)).toMatchObject({
|
||||
success: false,
|
||||
attempt: 1,
|
||||
finalError: "Retry cancelled",
|
||||
});
|
||||
});
|
||||
|
||||
it("exhausts max retries and emits a failure event", async () => {
|
||||
const harness = await createHarness({ settings: { retry: { enabled: true, maxRetries: 2, baseDelayMs: 1 } } });
|
||||
harnesses.push(harness);
|
||||
@@ -225,6 +251,10 @@ describe("AgentSession retry and event characterization", () => {
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(order).toEqual([
|
||||
"extension:message_start:system",
|
||||
"public:message_start:system",
|
||||
"extension:message_end:system",
|
||||
"public:message_end:system",
|
||||
"extension:message_start:user",
|
||||
"public:message_start:user",
|
||||
"extension:message_end:user",
|
||||
@@ -246,6 +276,8 @@ describe("AgentSession retry and event characterization", () => {
|
||||
expect(normalizeEventOrder(harness.events)).toEqual([
|
||||
"agent_start",
|
||||
"turn_start",
|
||||
"message_start:system",
|
||||
"message_end:system",
|
||||
"message_start:user",
|
||||
"message_end:user",
|
||||
"message_start:assistant",
|
||||
@@ -283,6 +315,8 @@ describe("AgentSession retry and event characterization", () => {
|
||||
expect(normalizeEventOrder(harness.events)).toEqual([
|
||||
"agent_start",
|
||||
"turn_start",
|
||||
"message_start:system",
|
||||
"message_end:system",
|
||||
"message_start:user",
|
||||
"message_end:user",
|
||||
"message_start:assistant",
|
||||
|
||||
@@ -205,6 +205,7 @@ describe("AgentSessionRuntime characterization", () => {
|
||||
.getEntries()
|
||||
.filter((entry) => entry.type === "message");
|
||||
expect(outgoingEntries.map((entry) => entry.message.role)).toEqual([
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
@@ -383,7 +384,7 @@ describe("AgentSessionRuntime characterization", () => {
|
||||
expect(leafId).toBeTruthy();
|
||||
|
||||
await expect(runtime.fork(leafId!, { position: "at" })).rejects.toThrow(
|
||||
"This session has not been saved yet. Wait for the first assistant response before cloning or forking it.",
|
||||
"This session has not been saved yet. Send a message before cloning or forking it.",
|
||||
);
|
||||
});
|
||||
|
||||
|
||||
145
third_party/pi/packages/coding-agent/test/suite/agent-session-tool-orchestration.test.ts
vendored
Normal file
145
third_party/pi/packages/coding-agent/test/suite/agent-session-tool-orchestration.test.ts
vendored
Normal file
@@ -0,0 +1,145 @@
|
||||
import { fauxAssistantMessage, fauxToolCall, getCurrentTools, type TranscriptContext } from "@earendil-works/pi-ai";
|
||||
import type { ToolResultMessage } from "@earendil-works/pi-ai/compat";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { ExtensionAPI } from "../../src/core/extensions/types.ts";
|
||||
import { createCodemodeExtension } from "../../src/extensions/codemode/index.ts";
|
||||
import { createToolSearchExtension } from "../../src/extensions/tool-search/index.ts";
|
||||
import { createHarness, type Harness } from "./harness.ts";
|
||||
|
||||
/**
|
||||
* A tool that calls other tools, built only on the extension API: its own name, exposure, loadout
|
||||
* hook, and ctx.executeTool(). Codemode and tool search use the same mechanisms.
|
||||
*/
|
||||
function orchestratorExtension(pi: ExtensionAPI): void {
|
||||
pi.registerTool({
|
||||
name: "echo",
|
||||
label: "echo",
|
||||
description: "Echo text.",
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async (_id, { text }) => ({ content: [{ type: "text", text: `echo: ${text}` }], details: {} }),
|
||||
});
|
||||
pi.registerTool({
|
||||
name: "helper",
|
||||
label: "helper",
|
||||
description: "Only reachable from other tools.",
|
||||
parameters: Type.Object({}),
|
||||
exposure: "codemode",
|
||||
execute: async () => ({ content: [{ type: "text", text: "helped" }], details: {} }),
|
||||
});
|
||||
pi.registerTool({
|
||||
name: "run_tools",
|
||||
label: "run_tools",
|
||||
description: "Runs tools.",
|
||||
parameters: Type.Object({}),
|
||||
exposure: "model-only",
|
||||
prepareLoadout: (loadout) => ({
|
||||
descriptions: {
|
||||
run_tools: `Runs tools: ${loadout.callable.map((tool) => tool.name).join(", ")}`,
|
||||
echo: "Echo text (also callable from run_tools).",
|
||||
},
|
||||
hiddenDeclarations: ["echo"],
|
||||
}),
|
||||
execute: async (_id, _params, _signal, _onUpdate, ctx) => {
|
||||
const helper = await ctx.executeTool("helper", {});
|
||||
const echo = await ctx.executeTool("echo", { text: "hi" });
|
||||
const self = await ctx.executeTool("run_tools", {});
|
||||
const text = [helper, echo, self]
|
||||
.map((outcome) => (outcome.result.content[0] as { text: string }).text)
|
||||
.join(" | ");
|
||||
return { content: [{ type: "text", text }], details: {} };
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
describe("AgentSession tool orchestration", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (harnesses.length > 0) harnesses.pop()?.cleanup();
|
||||
});
|
||||
|
||||
it("supports tools that call other tools under any name through the extension API", async () => {
|
||||
const toolCalls: string[] = [];
|
||||
const harness = await createHarness({
|
||||
initialActiveToolNames: [],
|
||||
extensionFactories: [
|
||||
orchestratorExtension,
|
||||
(pi) => {
|
||||
pi.on("tool_call", (event) => {
|
||||
toolCalls.push(`${event.toolName}:${event.parentToolCallId ?? "top"}`);
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
await harness.session.bindExtensions({});
|
||||
|
||||
expect(harness.session.getActiveToolNames()).toEqual(["echo", "run_tools"]);
|
||||
expect(harness.session.getCallableToolNames()).toEqual(["echo", "helper"]);
|
||||
const find = (name: string) => harness.session.agent.state.tools.find((tool) => tool.name === name);
|
||||
expect(find("run_tools")?.description).toBe("Runs tools: echo, helper");
|
||||
expect(find("echo")?.description).toBe("Echo text (also callable from run_tools).");
|
||||
|
||||
const requestTools: string[][] = [];
|
||||
harness.setResponses([
|
||||
(context: TranscriptContext) => {
|
||||
requestTools.push(getCurrentTools(context.messages).map((tool) => tool.name));
|
||||
return fauxAssistantMessage([fauxToolCall("run_tools", {})], { stopReason: "toolUse" });
|
||||
},
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
await harness.session.prompt("go");
|
||||
|
||||
// echo stays active, but its declaration is left out of requests.
|
||||
expect(requestTools[0]).toEqual(["run_tools"]);
|
||||
const result = harness.session.messages.find(
|
||||
(message): message is ToolResultMessage => message.role === "toolResult",
|
||||
);
|
||||
if (!result) throw new Error("No tool result");
|
||||
expect(result.content).toEqual([{ type: "text", text: "helped | echo: hi | Tool run_tools not found" }]);
|
||||
const parent = result.toolCallId;
|
||||
expect(toolCalls).toEqual(["run_tools:top", `helper:${parent}`, `echo:${parent}`]);
|
||||
expect(result.nestedCalls?.calls.map((call) => [call.id, call.name, call.status])).toEqual([
|
||||
[`${parent}/1`, "helper", "ok"],
|
||||
[`${parent}/2`, "echo", "ok"],
|
||||
[`${parent}/3`, "run_tools", "error"],
|
||||
]);
|
||||
// The record is persisted with the session.
|
||||
const persisted = harness.sessionManager
|
||||
.getBranch()
|
||||
.find((entry) => entry.type === "message" && entry.message.role === "toolResult");
|
||||
expect(persisted?.type === "message" && persisted.message).toMatchObject({ nestedCalls: result.nestedCalls });
|
||||
});
|
||||
|
||||
it("registers codemode and tool_search inactive until they are named", async () => {
|
||||
const extensionFactories = [createCodemodeExtension(), createToolSearchExtension()];
|
||||
const plain = await createHarness({ extensionFactories });
|
||||
harnesses.push(plain);
|
||||
expect(plain.session.getAllTools().map((tool) => tool.name)).toEqual(
|
||||
expect.arrayContaining(["codemode", "tool_search"]),
|
||||
);
|
||||
expect(plain.session.getActiveToolNames()).toEqual(["read", "bash", "edit", "write"]);
|
||||
|
||||
// --tools and the defaultTools setting name them explicitly.
|
||||
const allowed = await createHarness({ allowedToolNames: ["read", "codemode"], extensionFactories });
|
||||
harnesses.push(allowed);
|
||||
expect(allowed.session.getActiveToolNames()).toEqual(["read", "codemode"]);
|
||||
const initial = await createHarness({ initialActiveToolNames: ["tool_search"], extensionFactories });
|
||||
harnesses.push(initial);
|
||||
expect(initial.session.getActiveToolNames()).toEqual(["tool_search"]);
|
||||
});
|
||||
|
||||
it("leaves results without nested calls unchanged", async () => {
|
||||
const harness = await createHarness({ initialActiveToolNames: [], extensionFactories: [orchestratorExtension] });
|
||||
harnesses.push(harness);
|
||||
await harness.session.bindExtensions({});
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage([fauxToolCall("echo", { text: "x" })], { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
await harness.session.prompt("go");
|
||||
const result = harness.session.messages.find((message) => message.role === "toolResult");
|
||||
expect(result && "nestedCalls" in result).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -34,12 +34,15 @@ describe("AgentSession tool result images", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("passes images.autoResize to tool result normalization", async () => {
|
||||
it("passes image settings and the current model profile to tool result normalization", async () => {
|
||||
const resizeOptions = { maxWidth: 1200, maxHeight: 1000, maxBytes: 500000, jpegQuality: 70 };
|
||||
const harness = await createHarness({
|
||||
tools: [screenshotTool],
|
||||
settings: { images: { autoResize: false } },
|
||||
});
|
||||
harnesses.push(harness);
|
||||
if (!harness.session.model) throw new Error("Expected a model");
|
||||
harness.session.model.inputLimits = { images: { resize: resizeOptions } };
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage([fauxToolCall("screenshot", {})], { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
@@ -47,6 +50,9 @@ describe("AgentSession tool result images", () => {
|
||||
|
||||
await harness.session.prompt("take a screenshot");
|
||||
|
||||
expect(normalizeToolResultImages).toHaveBeenCalledWith(expect.any(Array), { autoResizeImages: false });
|
||||
expect(normalizeToolResultImages).toHaveBeenCalledWith(expect.any(Array), {
|
||||
autoResizeImages: false,
|
||||
resizeOptions,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -13,16 +13,18 @@ import type {
|
||||
FauxProviderRegistration,
|
||||
FauxResponseStep,
|
||||
Model,
|
||||
ToolResultMessage,
|
||||
} from "@earendil-works/pi-ai/compat";
|
||||
import { registerFauxProvider, streamSimple } from "@earendil-works/pi-ai/compat";
|
||||
import { AgentSession, type AgentSessionEvent } from "../../src/core/agent-session.ts";
|
||||
import { AuthStorage } from "../../src/core/auth-storage.ts";
|
||||
import type { ExtensionRunner } from "../../src/core/extensions/index.ts";
|
||||
import type { ExtensionRunner, ExtensionUIContext } from "../../src/core/extensions/index.ts";
|
||||
import { convertToLlm } from "../../src/core/messages.ts";
|
||||
import { SessionManager } from "../../src/core/session-manager.ts";
|
||||
import type { Settings } from "../../src/core/settings-manager.ts";
|
||||
import { SettingsManager } from "../../src/core/settings-manager.ts";
|
||||
import type { InlineExtension, ResourceLoader } from "../../src/index.ts";
|
||||
import { theme } from "../../src/modes/interactive/theme/theme.ts";
|
||||
import {
|
||||
type CreateTestExtensionsResultInput,
|
||||
createTestExtensionsResult,
|
||||
@@ -60,10 +62,55 @@ export function getAssistantTexts(harness: Harness): string[] {
|
||||
.map((message) => getMessageText(message));
|
||||
}
|
||||
|
||||
/** The latest result of `toolName` in the session transcript. */
|
||||
export function getToolResult(harness: Harness, toolName: string): ToolResultMessage {
|
||||
const result = harness.session.messages.findLast(
|
||||
(message): message is ToolResultMessage => message.role === "toolResult" && message.toolName === toolName,
|
||||
);
|
||||
if (!result) throw new Error(`No ${toolName} tool result`);
|
||||
return result;
|
||||
}
|
||||
|
||||
/** An extension UI context that does nothing, with `overrides` applied. */
|
||||
export function createTestUiContext(overrides: Partial<ExtensionUIContext> = {}): ExtensionUIContext {
|
||||
return {
|
||||
select: async () => undefined,
|
||||
confirm: async () => false,
|
||||
input: async () => undefined,
|
||||
notify: () => {},
|
||||
onTerminalInput: () => () => {},
|
||||
setStatus: () => {},
|
||||
setWorkingMessage: () => {},
|
||||
setWorkingVisible: () => {},
|
||||
setWorkingIndicator: () => {},
|
||||
setHiddenThinkingLabel: () => {},
|
||||
setWidget: () => {},
|
||||
setFooter: () => {},
|
||||
setHeader: () => {},
|
||||
setTitle: () => {},
|
||||
custom: async <T>() => undefined as T,
|
||||
pasteToEditor: () => {},
|
||||
setEditorText: () => {},
|
||||
getEditorText: () => "",
|
||||
editor: async () => undefined,
|
||||
addAutocompleteProvider: () => {},
|
||||
setEditorComponent: () => {},
|
||||
getEditorComponent: () => undefined,
|
||||
get theme() {
|
||||
return theme;
|
||||
},
|
||||
getAllThemes: () => [],
|
||||
getTheme: () => undefined,
|
||||
setTheme: () => ({ success: false, error: "Theme switching not available in tests" }),
|
||||
getToolsExpanded: () => false,
|
||||
setToolsExpanded: () => {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export interface HarnessOptions {
|
||||
models?: FauxModelDefinition[];
|
||||
settings?: Partial<Settings>;
|
||||
systemPrompt?: string;
|
||||
tools?: AgentTool[];
|
||||
initialActiveToolNames?: string[];
|
||||
allowedToolNames?: string[];
|
||||
@@ -72,6 +119,8 @@ export interface HarnessOptions {
|
||||
extensionFactories?: Array<InlineExtension | CreateTestExtensionsResultInput>;
|
||||
withConfiguredAuth?: boolean;
|
||||
modelsJson?: Record<string, unknown>;
|
||||
/** Session to continue, for example to test a resume. Default: a new in-memory session. */
|
||||
sessionManager?: SessionManager;
|
||||
}
|
||||
|
||||
export interface Harness {
|
||||
@@ -109,7 +158,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise<Harne
|
||||
const withConfiguredAuth = options.withConfiguredAuth ?? true;
|
||||
const extensionRunnerRef: { current?: ExtensionRunner } = {};
|
||||
|
||||
const sessionManager = SessionManager.inMemory();
|
||||
const sessionManager = options.sessionManager ?? SessionManager.inMemory();
|
||||
const settingsManager = SettingsManager.inMemory(options.settings);
|
||||
|
||||
const authStorage = AuthStorage.inMemory();
|
||||
@@ -132,6 +181,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise<Harne
|
||||
api: registeredModel.api,
|
||||
reasoning: registeredModel.reasoning,
|
||||
input: registeredModel.input,
|
||||
inputLimits: registeredModel.inputLimits,
|
||||
cost: registeredModel.cost,
|
||||
contextWindow: registeredModel.contextWindow,
|
||||
maxTokens: registeredModel.maxTokens,
|
||||
@@ -145,7 +195,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise<Harne
|
||||
streamFn: streamSimple,
|
||||
initialState: {
|
||||
model,
|
||||
systemPrompt: options.systemPrompt ?? "You are a test assistant.",
|
||||
systemPrompt: "",
|
||||
tools: [],
|
||||
},
|
||||
convertToLlm,
|
||||
@@ -188,6 +238,7 @@ export async function createHarness(options: HarnessOptions = {}): Promise<Harne
|
||||
resourceLoader,
|
||||
baseToolsOverride: toolMap,
|
||||
initialActiveToolNames: options.initialActiveToolNames,
|
||||
usesDefaultTools: options.initialActiveToolNames === undefined,
|
||||
allowedToolNames: options.allowedToolNames,
|
||||
excludedToolNames: options.excludedToolNames,
|
||||
extensionRunnerRef,
|
||||
|
||||
182
third_party/pi/packages/coding-agent/test/suite/mcp-oauth-server.ts
vendored
Normal file
182
third_party/pi/packages/coding-agent/test/suite/mcp-oauth-server.ts
vendored
Normal file
@@ -0,0 +1,182 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { createServer, type IncomingMessage, type Server, type ServerResponse } from "node:http";
|
||||
import { LATEST_PROTOCOL_VERSION } from "@earendil-works/pi-mcp";
|
||||
|
||||
async function readBody(request: IncomingMessage): Promise<string> {
|
||||
const chunks: Buffer[] = [];
|
||||
for await (const chunk of request) chunks.push(Buffer.from(chunk));
|
||||
return Buffer.concat(chunks).toString("utf8");
|
||||
}
|
||||
|
||||
function json(response: ServerResponse, status: number, body: unknown, headers: Record<string, string> = {}): void {
|
||||
response.writeHead(status, { "content-type": "application/json", ...headers }).end(JSON.stringify(body));
|
||||
}
|
||||
|
||||
/**
|
||||
* MCP server protected by OAuth, with its own authorization server (discovery, DCR, PKCE, refresh).
|
||||
* `iss` is sent as the `iss` parameter of authorization responses (RFC 9207). `issParameter` advertises
|
||||
* that parameter and sends the server's issuer. `cimd` advertises Client ID Metadata Documents.
|
||||
* `redirectPath` replaces the path of the redirect URI, like a mixed-up authorization server.
|
||||
* The MCP endpoint assigns a session, so closing a connection sends a DELETE. Paths added to `stall`
|
||||
* accept requests and never answer them, like an unresponsive server.
|
||||
*/
|
||||
export async function startOAuthMcpServer(
|
||||
options: { iss?: string; issParameter?: boolean; cimd?: boolean; redirectPath?: string } = {},
|
||||
) {
|
||||
const log: string[] = [];
|
||||
const registrations: Record<string, unknown>[] = [];
|
||||
const authorizations: URLSearchParams[] = [];
|
||||
const tokenRequests: URLSearchParams[] = [];
|
||||
const deletes: (string | undefined)[] = [];
|
||||
const stall = new Set<string>();
|
||||
const stalled: { path: string; request: IncomingMessage }[] = [];
|
||||
const validTokens = new Set<string>();
|
||||
const refreshTokens = new Set<string>();
|
||||
const challenges = new Map<string, string>();
|
||||
let issued = 0;
|
||||
let origin = "";
|
||||
|
||||
const issueTokens = () => {
|
||||
issued++;
|
||||
const tokens = { access_token: `access-${issued}`, refresh_token: `refresh-${issued}` };
|
||||
validTokens.add(tokens.access_token);
|
||||
refreshTokens.add(tokens.refresh_token);
|
||||
return { ...tokens, token_type: "Bearer", expires_in: 3600 };
|
||||
};
|
||||
|
||||
const handleMcp = async (request: IncomingMessage, response: ServerResponse) => {
|
||||
const token = request.headers.authorization?.replace(/^Bearer /, "");
|
||||
if (request.method === "DELETE") deletes.push(token);
|
||||
if (request.method !== "POST") {
|
||||
response.writeHead(request.method === "GET" ? 405 : 200).end();
|
||||
return;
|
||||
}
|
||||
if (!token || !validTokens.has(token)) {
|
||||
log.push(`401 ${token ?? "none"}`);
|
||||
response
|
||||
.writeHead(401, {
|
||||
"www-authenticate": `Bearer resource_metadata="${origin}/.well-known/oauth-protected-resource/mcp"`,
|
||||
})
|
||||
.end();
|
||||
return;
|
||||
}
|
||||
const message = JSON.parse(await readBody(request)) as { id?: number; method: string; params?: unknown };
|
||||
if (message.id === undefined) {
|
||||
response.writeHead(202).end();
|
||||
return;
|
||||
}
|
||||
let result: unknown;
|
||||
if (message.method === "initialize") {
|
||||
result = {
|
||||
protocolVersion: LATEST_PROTOCOL_VERSION,
|
||||
capabilities: { tools: {} },
|
||||
serverInfo: { name: "issues", version: "1.0.0" },
|
||||
};
|
||||
} else if (message.method === "tools/list") {
|
||||
result = { tools: [{ name: "whoami", inputSchema: { type: "object", properties: {} } }] };
|
||||
} else if (message.method === "tools/call") {
|
||||
log.push(`call ${token}`);
|
||||
result = { content: [{ type: "text", text: `token ${token}` }] };
|
||||
} else {
|
||||
result = {};
|
||||
}
|
||||
json(response, 200, { jsonrpc: "2.0", id: message.id, result }, { "mcp-session-id": "session-1" });
|
||||
};
|
||||
|
||||
const handle = async (request: IncomingMessage, response: ServerResponse) => {
|
||||
const url = new URL(request.url ?? "/", origin);
|
||||
if (stall.has(url.pathname)) {
|
||||
stalled.push({ path: url.pathname, request });
|
||||
return;
|
||||
}
|
||||
switch (url.pathname) {
|
||||
case "/mcp":
|
||||
return handleMcp(request, response);
|
||||
case "/.well-known/oauth-protected-resource/mcp":
|
||||
return json(response, 200, { resource: `${origin}/mcp`, authorization_servers: [origin] });
|
||||
case "/.well-known/oauth-authorization-server":
|
||||
return json(response, 200, {
|
||||
issuer: origin,
|
||||
authorization_endpoint: `${origin}/authorize`,
|
||||
token_endpoint: `${origin}/token`,
|
||||
registration_endpoint: `${origin}/register`,
|
||||
response_types_supported: ["code"],
|
||||
code_challenge_methods_supported: ["S256"],
|
||||
token_endpoint_auth_methods_supported: ["none"],
|
||||
...(options.cimd ? { client_id_metadata_document_supported: true } : {}),
|
||||
...(options.issParameter ? { authorization_response_iss_parameter_supported: true } : {}),
|
||||
});
|
||||
case "/register": {
|
||||
const metadata = JSON.parse(await readBody(request)) as Record<string, unknown>;
|
||||
log.push("register");
|
||||
registrations.push(metadata);
|
||||
return json(response, 201, { ...metadata, client_id: "client-1" });
|
||||
}
|
||||
case "/authorize": {
|
||||
authorizations.push(url.searchParams);
|
||||
const code = `code-${challenges.size + 1}`;
|
||||
challenges.set(code, url.searchParams.get("code_challenge") ?? "");
|
||||
const redirect = new URL(url.searchParams.get("redirect_uri") ?? "");
|
||||
if (options.redirectPath) redirect.pathname = options.redirectPath;
|
||||
redirect.searchParams.set("code", code);
|
||||
redirect.searchParams.set("state", url.searchParams.get("state") ?? "");
|
||||
const iss = options.iss ?? (options.issParameter ? origin : undefined);
|
||||
if (iss) redirect.searchParams.set("iss", iss);
|
||||
response.writeHead(302, { location: redirect.href }).end();
|
||||
return;
|
||||
}
|
||||
case "/token": {
|
||||
const params = new URLSearchParams(await readBody(request));
|
||||
tokenRequests.push(params);
|
||||
if (params.get("grant_type") === "authorization_code") {
|
||||
const challenge = challenges.get(params.get("code") ?? "");
|
||||
const verifier = createHash("sha256")
|
||||
.update(params.get("code_verifier") ?? "")
|
||||
.digest("base64url");
|
||||
if (!challenge || challenge !== verifier) return json(response, 400, { error: "invalid_grant" });
|
||||
challenges.delete(params.get("code") ?? "");
|
||||
log.push("token code");
|
||||
return json(response, 200, issueTokens());
|
||||
}
|
||||
const refresh = params.get("refresh_token") ?? "";
|
||||
if (!refreshTokens.delete(refresh)) return json(response, 400, { error: "invalid_grant" });
|
||||
log.push("token refresh");
|
||||
return json(response, 200, issueTokens());
|
||||
}
|
||||
default:
|
||||
response.writeHead(404).end();
|
||||
}
|
||||
};
|
||||
|
||||
const server: Server = createServer((request, response) => {
|
||||
void handle(request, response).catch((error) => {
|
||||
response.writeHead(500).end(String(error));
|
||||
});
|
||||
});
|
||||
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
const address = server.address();
|
||||
if (!address || typeof address === "string") throw new Error("test server did not bind to TCP");
|
||||
origin = `http://127.0.0.1:${address.port}`;
|
||||
return {
|
||||
url: `${origin}/mcp`,
|
||||
log,
|
||||
/** Client metadata of dynamic client registrations. */
|
||||
registrations,
|
||||
/** Query parameters of authorization requests. */
|
||||
authorizations,
|
||||
/** Parameters of token requests. */
|
||||
tokenRequests,
|
||||
/** Access tokens of session DELETE requests. */
|
||||
deletes,
|
||||
stall,
|
||||
/** Requests to stalled paths, still open unless the client gave up. */
|
||||
stalled,
|
||||
/** Simulates access token expiry. */
|
||||
expireAccessTokens: () => validTokens.clear(),
|
||||
close: () =>
|
||||
new Promise<void>((resolve) => {
|
||||
server.closeAllConnections();
|
||||
server.close(() => resolve());
|
||||
}),
|
||||
};
|
||||
}
|
||||
72
third_party/pi/packages/coding-agent/test/suite/regressions/10285-mcp-tool-renderers.test.ts
vendored
Normal file
72
third_party/pi/packages/coding-agent/test/suite/regressions/10285-mcp-tool-renderers.test.ts
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
import { mkdtempSync, readFileSync, rmSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ToolRenderContext } from "../../../src/core/extensions/types.ts";
|
||||
import { SessionManager } from "../../../src/core/session-manager.ts";
|
||||
import { createMcpExtension } from "../../../src/extensions/mcp/index.ts";
|
||||
import { initTheme, theme } from "../../../src/modes/interactive/theme/theme.ts";
|
||||
import { stripAnsi } from "../../../src/utils/ansi.ts";
|
||||
import { createHarness } from "../harness.ts";
|
||||
|
||||
const mcpExtension = createMcpExtension({ loadConfig: () => ({ servers: [], errors: [] }) });
|
||||
|
||||
describe("MCP tool renderers", () => {
|
||||
// Regression #10285: a resumed session renders MCP tool calls before their server connected, if it
|
||||
// ever does. They render with the MCP renderers anyway, instead of the expanded fallback.
|
||||
it("renders calls to MCP tools that are not registered", async () => {
|
||||
initTheme("dark");
|
||||
const harness = await createHarness({ extensionFactories: [mcpExtension] });
|
||||
try {
|
||||
const resolve = (toolName: string) =>
|
||||
harness.session.extensionRunner.resolveToolRenderers(toolName, () =>
|
||||
harness.session.getToolDefinition(toolName),
|
||||
);
|
||||
|
||||
const call = resolve("mcp__my_docs__search")?.renderCall?.({ query: "pi" }, theme, {
|
||||
expanded: false,
|
||||
} as unknown as ToolRenderContext);
|
||||
expect(stripAnsi(call?.render(100).join("\n") ?? "")).toContain('my_docs/search query="pi"');
|
||||
expect(resolve("not_mcp")).toBeUndefined();
|
||||
// Registered tools keep their own renderers.
|
||||
expect(resolve("read")?.renderCall).toBe(harness.session.getToolDefinition("read")?.renderCall);
|
||||
} finally {
|
||||
harness.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("renders them in HTML exports too", async () => {
|
||||
initTheme("dark");
|
||||
const dir = mkdtempSync(join(tmpdir(), "pi-10285-"));
|
||||
const sessionManager = SessionManager.create(dir, join(dir, "sessions"));
|
||||
const harness = await createHarness({ extensionFactories: [mcpExtension], sessionManager });
|
||||
try {
|
||||
sessionManager.appendMessage({ role: "user", content: "search", timestamp: 1 });
|
||||
sessionManager.appendMessage({
|
||||
role: "assistant",
|
||||
content: [{ type: "toolCall", id: "call-1", name: "mcp__my_docs__search", arguments: { query: "pi" } }],
|
||||
api: "anthropic-messages",
|
||||
provider: "anthropic",
|
||||
model: "test",
|
||||
usage: {
|
||||
input: 0,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 0,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
stopReason: "toolUse",
|
||||
timestamp: 2,
|
||||
});
|
||||
|
||||
const html = readFileSync(await harness.session.exportToHtml(join(dir, "export.html")), "utf8");
|
||||
const data = /<script id="session-data" type="application\/json">([^<]*)<\/script>/.exec(html)?.[1] ?? "";
|
||||
const session = JSON.parse(Buffer.from(data, "base64").toString("utf8"));
|
||||
expect(stripAnsi(session.renderedTools?.["call-1"]?.callHtml ?? "")).toContain("my_docs/search");
|
||||
} finally {
|
||||
harness.cleanup();
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
});
|
||||
@@ -53,6 +53,7 @@ describe("regressions #1717/#2113: agent session event settlement", () => {
|
||||
.filter((entry) => entry.type === "message")
|
||||
.map((entry) => entry.message);
|
||||
expect(branchMessages.map((message) => message.role)).toEqual([
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
@@ -90,6 +91,6 @@ describe("regressions #1717/#2113: agent session event settlement", () => {
|
||||
|
||||
await harness.session.prompt("run tool");
|
||||
|
||||
expect(branchRolesAtToolCall).toEqual([["user", "assistant"]]);
|
||||
expect(branchRolesAtToolCall).toEqual([["system", "user", "assistant"]]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ import { join } from "node:path";
|
||||
import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
|
||||
/**
|
||||
* Regression test for https://github.com/earendil-works/pi-mono/issues/2791
|
||||
* Regression test for https://github.com/earendil-works/pi/issues/2791
|
||||
*
|
||||
* fs.watch() returns an FSWatcher (EventEmitter). If the watcher emits an
|
||||
* 'error' event after creation and no error handler is attached, Node.js
|
||||
|
||||
@@ -87,7 +87,7 @@ describe("regression #2835: tool allowlists filter extension tools", () => {
|
||||
|
||||
expect(session.getAllTools()).toEqual([]);
|
||||
expect(session.getActiveToolNames()).toEqual([]);
|
||||
expect(session.systemPrompt).toContain("Available tools:\n(none)");
|
||||
expect(session.systemPrompt).toContain("<tools>\n(none)\n");
|
||||
expect(session.systemPrompt).not.toContain("dynamic_tool");
|
||||
session.dispose();
|
||||
});
|
||||
|
||||
@@ -201,10 +201,11 @@ describe("regression #2860: replaced session callbacks", () => {
|
||||
expect(replacementSessionFile).not.toBe(oldSessionFile);
|
||||
expect(staleCtxThrows).toBe(true);
|
||||
expect(stalePiThrows).toBe(true);
|
||||
expect(runtime.session.messages.map((message) => `${message.role}:${getText(message)}`)).toEqual([
|
||||
"user:Hello from the new session!",
|
||||
"assistant:hello reply",
|
||||
]);
|
||||
expect(
|
||||
runtime.session.messages
|
||||
.filter((message) => message.role !== "system")
|
||||
.map((message) => `${message.role}:${getText(message)}`),
|
||||
).toEqual(["user:Hello from the new session!", "assistant:hello reply"]);
|
||||
});
|
||||
|
||||
it("supports withSession for fork", async () => {
|
||||
@@ -232,12 +233,11 @@ describe("regression #2860: replaced session callbacks", () => {
|
||||
await runtime.session.prompt("seed");
|
||||
await runtime.session.prompt("/fork-it");
|
||||
|
||||
expect(runtime.session.messages.map((message) => `${message.role}:${getText(message)}`)).toEqual([
|
||||
"user:seed",
|
||||
"assistant:seed reply",
|
||||
"user:fork callback message",
|
||||
"assistant:fork reply",
|
||||
]);
|
||||
expect(
|
||||
runtime.session.messages
|
||||
.filter((message) => message.role !== "system")
|
||||
.map((message) => `${message.role}:${getText(message)}`),
|
||||
).toEqual(["user:seed", "assistant:seed reply", "user:fork callback message", "assistant:fork reply"]);
|
||||
});
|
||||
|
||||
it("supports withSession for switchSession", async () => {
|
||||
@@ -269,11 +269,10 @@ describe("regression #2860: replaced session callbacks", () => {
|
||||
await runtime.session.prompt("/switch-it");
|
||||
|
||||
expect(runtime.session.sessionFile).toBe(targetSessionPath);
|
||||
expect(runtime.session.messages.map((message) => `${message.role}:${getText(message)}`)).toEqual([
|
||||
"user:target",
|
||||
"assistant:target reply",
|
||||
"user:switch callback message",
|
||||
"assistant:switch reply",
|
||||
]);
|
||||
expect(
|
||||
runtime.session.messages
|
||||
.filter((message) => message.role !== "system")
|
||||
.map((message) => `${message.role}:${getText(message)}`),
|
||||
).toEqual(["user:target", "assistant:target reply", "user:switch callback message", "assistant:switch reply"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createFindToolDefinition } from "../../../src/core/tools/find.ts";
|
||||
|
||||
/**
|
||||
* Regression test for https://github.com/earendil-works/pi-mono/issues/3302
|
||||
* Regression test for https://github.com/earendil-works/pi/issues/3302
|
||||
*
|
||||
* The `find` tool advertises glob patterns like `src/**\/*.spec.ts`, but the
|
||||
* default fd-backed implementation used `fd --glob <pattern>` without
|
||||
|
||||
@@ -5,7 +5,7 @@ import { afterEach, beforeEach, describe, expect, it } from "vitest";
|
||||
import { createFindToolDefinition } from "../../../src/core/tools/find.ts";
|
||||
|
||||
/**
|
||||
* Regression test for https://github.com/earendil-works/pi-mono/issues/3303
|
||||
* Regression test for https://github.com/earendil-works/pi/issues/3303
|
||||
*
|
||||
* The `find` tool previously collected every `.gitignore` under the search
|
||||
* path and passed them to `fd` via `--ignore-file`. fd treats `--ignore-file`
|
||||
|
||||
@@ -91,7 +91,7 @@ describe("regression #3592: no-builtin-tools keeps extension tools enabled", ()
|
||||
|
||||
expect(session.getAllTools()).toEqual([]);
|
||||
expect(session.getActiveToolNames()).toEqual([]);
|
||||
expect(session.systemPrompt).toContain("Available tools:\n(none)");
|
||||
expect(session.systemPrompt).toContain("<tools>\n(none)\n");
|
||||
session.dispose();
|
||||
});
|
||||
|
||||
@@ -112,7 +112,7 @@ describe("regression #3592: no-builtin-tools keeps extension tools enabled", ()
|
||||
});
|
||||
|
||||
expect(session.getActiveToolNames()).toEqual([]);
|
||||
expect(session.systemPrompt).toContain("Available tools:\n(none)");
|
||||
expect(session.systemPrompt).toContain("<tools>\n(none)\n");
|
||||
expect(session.systemPrompt).not.toContain("- read:");
|
||||
session.dispose();
|
||||
});
|
||||
|
||||
@@ -37,6 +37,7 @@ type RenderSessionContextThis = {
|
||||
pendingTools: Map<string, ToolExecutionComponent>;
|
||||
chatContainer: Container;
|
||||
footer: { invalidate(): void };
|
||||
programStatus: { handleEvent(): void };
|
||||
ui: TUI;
|
||||
settingsManager: {
|
||||
getShowImages(): boolean;
|
||||
@@ -68,6 +69,7 @@ function createFakeInteractiveModeThis(): RenderSessionContextThis {
|
||||
pendingTools: new Map<string, ToolExecutionComponent>(),
|
||||
chatContainer,
|
||||
footer: { invalidate: vi.fn() },
|
||||
programStatus: { handleEvent: vi.fn() },
|
||||
ui: { requestRender: vi.fn() } as unknown as TUI,
|
||||
settingsManager: {
|
||||
getShowImages: () => false,
|
||||
|
||||
@@ -34,7 +34,13 @@ const interactiveModePrototype = InteractiveMode.prototype as unknown;
|
||||
const tempDirs: string[] = [];
|
||||
const originalStdoutIsTTY = Object.getOwnPropertyDescriptor(process.stdout, "isTTY");
|
||||
|
||||
class ProcessExitError extends Error {}
|
||||
class ProcessExitError extends Error {
|
||||
readonly code: number | undefined;
|
||||
constructor(code?: number) {
|
||||
super(`process.exit(${code})`);
|
||||
this.code = code;
|
||||
}
|
||||
}
|
||||
|
||||
function createSessionManager(options: { sessionFile?: string } = {}): SessionManager {
|
||||
return {
|
||||
@@ -183,3 +189,131 @@ describe("InteractiveMode.shutdown ordering (#5080)", () => {
|
||||
expect(context.runtimeHost.dispose).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
// Regression for the `read EIO` crash reports (crash_tty_read_eio).
|
||||
//
|
||||
// When the terminal goes away, stdin reads and setRawMode fail with EIO (pi is
|
||||
// left in an orphaned background process group) or ENOTTY (macOS revoked the
|
||||
// tty). Node emits these as `error` events on process.stdin. Without a stdin
|
||||
// listener they became uncaught exceptions that were recorded as crashes and
|
||||
// prompted users to run /bug on the next launch.
|
||||
|
||||
type HandlerContext = {
|
||||
isShuttingDown: boolean;
|
||||
signalCleanupHandlers: Array<() => void>;
|
||||
shutdown: () => Promise<void>;
|
||||
unregisterSignalHandlers: () => void;
|
||||
emergencyTerminalExit: () => never;
|
||||
uncaughtCrash: (error: Error) => never;
|
||||
recordCrash: () => boolean;
|
||||
getCrashExtensionHint: () => string | undefined;
|
||||
crashReportInstructions: () => string;
|
||||
ui: { stop: () => void };
|
||||
};
|
||||
|
||||
type InteractiveModePrivates = {
|
||||
registerSignalHandlers(this: HandlerContext): void;
|
||||
unregisterSignalHandlers(this: HandlerContext): void;
|
||||
emergencyTerminalExit(this: HandlerContext): never;
|
||||
uncaughtCrash(this: HandlerContext, error: Error): never;
|
||||
};
|
||||
|
||||
const proto = InteractiveMode.prototype as unknown as InteractiveModePrivates;
|
||||
|
||||
function createHandlerContext(): HandlerContext {
|
||||
const context: HandlerContext = {
|
||||
isShuttingDown: false,
|
||||
signalCleanupHandlers: [],
|
||||
shutdown: vi.fn(async () => {}),
|
||||
unregisterSignalHandlers: () => proto.unregisterSignalHandlers.call(context),
|
||||
emergencyTerminalExit: () => proto.emergencyTerminalExit.call(context),
|
||||
uncaughtCrash: (error: Error) => proto.uncaughtCrash.call(context, error),
|
||||
recordCrash: vi.fn(() => true),
|
||||
getCrashExtensionHint: () => undefined,
|
||||
crashReportInstructions: () => "",
|
||||
ui: { stop: vi.fn() },
|
||||
};
|
||||
return context;
|
||||
}
|
||||
|
||||
function errnoError(message: string, code: string): NodeJS.ErrnoException {
|
||||
return Object.assign(new Error(message), { code });
|
||||
}
|
||||
|
||||
function captureExit(fn: () => void): number | undefined {
|
||||
try {
|
||||
fn();
|
||||
} catch (error) {
|
||||
if (error instanceof ProcessExitError) return error.code;
|
||||
throw error;
|
||||
}
|
||||
throw new Error("expected process.exit");
|
||||
}
|
||||
|
||||
describe("dead terminal errors on stdin", () => {
|
||||
let context: HandlerContext | undefined;
|
||||
|
||||
afterEach(() => {
|
||||
context?.unregisterSignalHandlers();
|
||||
context = undefined;
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
function setup(): HandlerContext {
|
||||
vi.spyOn(process, "exit").mockImplementation(((code?: number) => {
|
||||
throw new ProcessExitError(code);
|
||||
}) as typeof process.exit);
|
||||
vi.spyOn(console, "error").mockImplementation(() => {});
|
||||
context = createHandlerContext();
|
||||
proto.registerSignalHandlers.call(context);
|
||||
return context;
|
||||
}
|
||||
|
||||
test.each([
|
||||
["read EIO", "EIO"],
|
||||
["setRawMode EIO", "EIO"],
|
||||
["setRawMode ENOTTY", "ENOTTY"],
|
||||
])("stdin %s exits quietly without recording a crash", (message, code) => {
|
||||
const ctx = setup();
|
||||
|
||||
const exitCode = captureExit(() => process.stdin.emit("error", errnoError(message, code)));
|
||||
|
||||
expect(exitCode).toBe(129);
|
||||
expect(ctx.recordCrash).not.toHaveBeenCalled();
|
||||
expect(ctx.ui.stop).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("other stdin errors still crash", () => {
|
||||
setup();
|
||||
const error = errnoError("read ECONNREFUSED", "ECONNREFUSED");
|
||||
|
||||
expect(() => process.stdin.emit("error", error)).toThrow(error);
|
||||
});
|
||||
|
||||
test("uncaught dead terminal errors are not recorded as crashes", () => {
|
||||
const ctx = setup();
|
||||
|
||||
const exitCode = captureExit(() => ctx.uncaughtCrash(errnoError("read EIO", "EIO")));
|
||||
|
||||
expect(exitCode).toBe(129);
|
||||
expect(ctx.recordCrash).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
test("other uncaught errors are still recorded as crashes", () => {
|
||||
const ctx = setup();
|
||||
|
||||
const exitCode = captureExit(() => ctx.uncaughtCrash(new Error("boom")));
|
||||
|
||||
expect(exitCode).toBe(1);
|
||||
expect(ctx.recordCrash).toHaveBeenCalledOnce();
|
||||
});
|
||||
|
||||
test("handlers are removed on unregister", () => {
|
||||
const ctx = setup();
|
||||
const before = process.stdin.listenerCount("error");
|
||||
|
||||
ctx.unregisterSignalHandlers();
|
||||
|
||||
expect(process.stdin.listenerCount("error")).toBe(before - 1);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -78,4 +78,22 @@ describe("regression #5109: exclude tools", () => {
|
||||
harness.cleanup();
|
||||
}
|
||||
});
|
||||
|
||||
it("matches allowlist and denylist patterns", async () => {
|
||||
const harness = await createHarness({
|
||||
allowedToolNames: ["*_tool", "ask_*", "re*"],
|
||||
excludedToolNames: ["ask*"],
|
||||
initialActiveToolNames: ["*_tool", "ask_*", "re*"],
|
||||
extensionFactories,
|
||||
});
|
||||
try {
|
||||
await harness.session.bindExtensions({});
|
||||
|
||||
// Patterns that match a tool activate it like its name.
|
||||
expect(toolNames(harness.session.getAllTools())).toEqual(["dynamic_tool", "read"]);
|
||||
expect(harness.session.getActiveToolNames().sort()).toEqual(["dynamic_tool", "read"]);
|
||||
} finally {
|
||||
harness.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
@@ -19,7 +19,13 @@ function createDialog(): LoginDialogComponent {
|
||||
}
|
||||
|
||||
function renderDialog(dialog: LoginDialogComponent): string[] {
|
||||
return stripAnsi(dialog.render(120).join("\n"))
|
||||
// stripAnsi keeps APC sequences, so drop the TUI's zero-width cursor markers (e.g. \x1b_pi:fc\x07) first
|
||||
return stripAnsi(
|
||||
dialog
|
||||
.render(120)
|
||||
.join("\n")
|
||||
.replace(/\x1b_[^\x07]*\x07/g, ""),
|
||||
)
|
||||
.split("\n")
|
||||
.map((line) => line.trimEnd());
|
||||
}
|
||||
|
||||
@@ -2,47 +2,9 @@ import { fauxAssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { Container, Text } from "@earendil-works/pi-tui";
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
import type { AgentSessionEvent } from "../../../src/core/agent-session.ts";
|
||||
import type { ExtensionUIContext } from "../../../src/core/extensions/index.ts";
|
||||
import { InteractiveMode } from "../../../src/modes/interactive/interactive-mode.ts";
|
||||
import { initTheme, type Theme, theme } from "../../../src/modes/interactive/theme/theme.ts";
|
||||
import { createHarness } from "../harness.ts";
|
||||
|
||||
function createUiContext(
|
||||
onNotify: (message: string, type: "info" | "warning" | "error" | undefined) => void,
|
||||
): ExtensionUIContext {
|
||||
return {
|
||||
select: async () => undefined,
|
||||
confirm: async () => false,
|
||||
input: async () => undefined,
|
||||
notify: onNotify,
|
||||
onTerminalInput: () => () => {},
|
||||
setStatus: () => {},
|
||||
setWorkingMessage: () => {},
|
||||
setWorkingVisible: () => {},
|
||||
setWorkingIndicator: () => {},
|
||||
setHiddenThinkingLabel: () => {},
|
||||
setWidget: () => {},
|
||||
setFooter: () => {},
|
||||
setHeader: () => {},
|
||||
setTitle: () => {},
|
||||
custom: async <T>() => undefined as T,
|
||||
pasteToEditor: () => {},
|
||||
setEditorText: () => {},
|
||||
getEditorText: () => "",
|
||||
editor: async () => undefined,
|
||||
addAutocompleteProvider: () => {},
|
||||
setEditorComponent: () => {},
|
||||
getEditorComponent: () => undefined,
|
||||
get theme() {
|
||||
return theme;
|
||||
},
|
||||
getAllThemes: () => [],
|
||||
getTheme: () => undefined,
|
||||
setTheme: (_theme: string | Theme) => ({ success: false, error: "Theme switching not available in tests" }),
|
||||
getToolsExpanded: () => false,
|
||||
setToolsExpanded: () => {},
|
||||
};
|
||||
}
|
||||
import { initTheme } from "../../../src/modes/interactive/theme/theme.ts";
|
||||
import { createHarness, createTestUiContext } from "../harness.ts";
|
||||
|
||||
type LoadedResourcesResult<T> = { [K in keyof T]: T[K] } & { diagnostics: [] };
|
||||
|
||||
@@ -70,6 +32,7 @@ type LoadedResourcesContext = {
|
||||
};
|
||||
};
|
||||
getStartupExpansionState: () => boolean;
|
||||
shouldShowStartupDetails: () => boolean;
|
||||
formatDisplayPath: (resourcePath: string) => string;
|
||||
formatContextPath: (resourcePath: string) => string;
|
||||
getBuiltInCommandConflictDiagnostics: (extensionRunner: LoadedResourcesContext["session"]["extensionRunner"]) => [];
|
||||
@@ -78,6 +41,7 @@ type LoadedResourcesContext = {
|
||||
type RebindContext = {
|
||||
unsubscribe?: () => void;
|
||||
applyRuntimeSettings: () => void;
|
||||
programStatus: { reset(): void };
|
||||
renderCurrentSessionState: () => void;
|
||||
bindCurrentSessionExtensions: () => Promise<void>;
|
||||
subscribeToAgent: () => void;
|
||||
@@ -246,6 +210,7 @@ function createLoadedResourcesContext(): LoadedResourcesContext {
|
||||
},
|
||||
},
|
||||
getStartupExpansionState: () => false,
|
||||
shouldShowStartupDetails: () => true,
|
||||
formatDisplayPath: (resourcePath) => resourcePath,
|
||||
formatContextPath: (resourcePath) => resourcePath.replace("/repo/", ""),
|
||||
getBuiltInCommandConflictDiagnostics: () => [],
|
||||
@@ -288,11 +253,12 @@ describe("regression #5943: session_start transient UI", () => {
|
||||
try {
|
||||
const context: RebindContext = {
|
||||
applyRuntimeSettings: () => events.push("apply"),
|
||||
programStatus: { reset: () => {} },
|
||||
renderCurrentSessionState: () => events.push("render"),
|
||||
bindCurrentSessionExtensions: async () => {
|
||||
events.push("bind");
|
||||
await harness.session.bindExtensions({
|
||||
uiContext: createUiContext((message) => events.push(`notify:${message}`)),
|
||||
uiContext: createTestUiContext({ notify: (message) => events.push(`notify:${message}`) }),
|
||||
mode: "tui",
|
||||
});
|
||||
},
|
||||
@@ -329,11 +295,12 @@ describe("regression #5943: session_start transient UI", () => {
|
||||
try {
|
||||
const context: RebindContext = {
|
||||
applyRuntimeSettings: () => {},
|
||||
programStatus: { reset: () => {} },
|
||||
renderCurrentSessionState: () => events.push("render"),
|
||||
bindCurrentSessionExtensions: async () => {
|
||||
events.push("bind");
|
||||
await harness.session.bindExtensions({
|
||||
uiContext: createUiContext(() => {}),
|
||||
uiContext: createTestUiContext(),
|
||||
mode: "tui",
|
||||
});
|
||||
},
|
||||
@@ -381,11 +348,12 @@ describe("regression #5943: session_start transient UI", () => {
|
||||
try {
|
||||
const context: RebindContext = {
|
||||
applyRuntimeSettings: () => {},
|
||||
programStatus: { reset: () => {} },
|
||||
renderCurrentSessionState: () => events.push("render"),
|
||||
bindCurrentSessionExtensions: async () => {
|
||||
events.push("bind");
|
||||
await harness.session.bindExtensions({
|
||||
uiContext: createUiContext(() => {}),
|
||||
uiContext: createTestUiContext(),
|
||||
mode: "tui",
|
||||
});
|
||||
},
|
||||
@@ -433,7 +401,7 @@ describe("regression #5943: session_start transient UI", () => {
|
||||
|
||||
try {
|
||||
await harness.session.bindExtensions({
|
||||
uiContext: createUiContext((message) => events.push(message)),
|
||||
uiContext: createTestUiContext({ notify: (message) => events.push(message) }),
|
||||
mode: "tui",
|
||||
});
|
||||
expect(events).toEqual(["start:startup", "notify:startup"]);
|
||||
|
||||
@@ -1,44 +1,48 @@
|
||||
import { fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
|
||||
import {
|
||||
fauxAssistantMessage,
|
||||
fauxToolCall,
|
||||
getCurrentSystemPrompt,
|
||||
getCurrentTools,
|
||||
type TranscriptContext,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { describe, expect, it } from "vitest";
|
||||
import type { ExtensionFactory } from "../../../src/index.ts";
|
||||
import type { ExtensionAPI, ExtensionFactory } from "../../../src/index.ts";
|
||||
import { createHarness } from "../harness.ts";
|
||||
|
||||
describe("extension active tools next-turn refresh", () => {
|
||||
it("applies pi.setActiveTools before the next provider request in the same run", async () => {
|
||||
const extensionFactories: ExtensionFactory[] = [
|
||||
(pi) => {
|
||||
pi.registerTool({
|
||||
name: "switch_tools",
|
||||
label: "Switch Tools",
|
||||
description: "Switch the active extension tool set",
|
||||
promptSnippet: "Switch to the next extension tool",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => {
|
||||
pi.setActiveTools(["after_switch"]);
|
||||
return {
|
||||
content: [{ type: "text", text: "switched" }],
|
||||
details: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
function getProviderToolNames(context: TranscriptContext): string[] {
|
||||
return getCurrentTools(context.messages)
|
||||
.map((tool) => tool.name)
|
||||
.sort();
|
||||
}
|
||||
|
||||
pi.registerTool({
|
||||
name: "after_switch",
|
||||
label: "After Switch",
|
||||
description: "Tool that should be available after switching",
|
||||
promptSnippet: "Run after the active tool set changes",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({
|
||||
content: [{ type: "text", text: "after" }],
|
||||
details: {},
|
||||
}),
|
||||
});
|
||||
},
|
||||
];
|
||||
const harness = await createHarness({
|
||||
extensionFactories,
|
||||
});
|
||||
/** Register `switch_tools`, which swaps the active set to `after_switch` when executed. */
|
||||
function registerSwitchTools(pi: ExtensionAPI): void {
|
||||
pi.registerTool({
|
||||
name: "switch_tools",
|
||||
label: "Switch Tools",
|
||||
description: "Switch the active extension tool set",
|
||||
promptSnippet: "Switch to the next extension tool",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => {
|
||||
pi.setActiveTools(["after_switch"]);
|
||||
return { content: [{ type: "text", text: "switched" }], details: {} };
|
||||
},
|
||||
});
|
||||
pi.registerTool({
|
||||
name: "after_switch",
|
||||
label: "After Switch",
|
||||
description: "Tool that should be available after switching",
|
||||
promptSnippet: "Run after the active tool set changes",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({ content: [{ type: "text", text: "after" }], details: {} }),
|
||||
});
|
||||
}
|
||||
|
||||
describe("extension active tools next-turn refresh", () => {
|
||||
// Regression #6162
|
||||
it("applies pi.setActiveTools before the next provider request in the same run", async () => {
|
||||
const harness = await createHarness({ extensionFactories: [registerSwitchTools] });
|
||||
|
||||
try {
|
||||
harness.session.setActiveToolsByName(["switch_tools"]);
|
||||
@@ -46,11 +50,11 @@ describe("extension active tools next-turn refresh", () => {
|
||||
const providerToolNames: string[][] = [];
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
providerToolNames.push((context.tools ?? []).map((tool) => tool.name).sort());
|
||||
providerToolNames.push(getProviderToolNames(context));
|
||||
return fauxAssistantMessage(fauxToolCall("switch_tools", {}), { stopReason: "toolUse" });
|
||||
},
|
||||
(context) => {
|
||||
providerToolNames.push((context.tools ?? []).map((tool) => tool.name).sort());
|
||||
providerToolNames.push(getProviderToolNames(context));
|
||||
return fauxAssistantMessage("done");
|
||||
},
|
||||
]);
|
||||
@@ -66,57 +70,30 @@ describe("extension active tools next-turn refresh", () => {
|
||||
}
|
||||
});
|
||||
|
||||
it("records additive active tool changes on the current tool result", async () => {
|
||||
const extensionFactories: ExtensionFactory[] = [
|
||||
(pi) => {
|
||||
pi.registerTool({
|
||||
name: "load_more_tools",
|
||||
label: "Load More Tools",
|
||||
description: "Load more tools",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => {
|
||||
pi.setActiveTools([...pi.getActiveTools(), "after_load"]);
|
||||
return {
|
||||
content: [{ type: "text", text: "loaded" }],
|
||||
details: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
pi.registerTool({
|
||||
name: "after_load",
|
||||
label: "After Load",
|
||||
description: "Tool available after loading",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({
|
||||
content: [{ type: "text", text: "after" }],
|
||||
details: {},
|
||||
}),
|
||||
});
|
||||
},
|
||||
];
|
||||
const harness = await createHarness({ extensionFactories });
|
||||
|
||||
it("reports the refreshed system prompt during the run", async () => {
|
||||
const harness = await createHarness({ extensionFactories: [registerSwitchTools] });
|
||||
try {
|
||||
harness.session.setActiveToolsByName(["load_more_tools"]);
|
||||
|
||||
const addedToolNames: string[][] = [];
|
||||
harness.session.setActiveToolsByName(["switch_tools"]);
|
||||
const providerPrompts: string[] = [];
|
||||
const sessionPrompts: string[] = [];
|
||||
harness.setResponses([
|
||||
() => fauxAssistantMessage(fauxToolCall("load_more_tools", {}), { stopReason: "toolUse" }),
|
||||
(context) => {
|
||||
addedToolNames.push(
|
||||
context.messages
|
||||
.filter((message) => message.role === "toolResult")
|
||||
.flatMap((message) => message.addedToolNames ?? []),
|
||||
);
|
||||
providerPrompts.push(getCurrentSystemPrompt(context.messages));
|
||||
sessionPrompts.push(harness.session.systemPrompt);
|
||||
return fauxAssistantMessage(fauxToolCall("switch_tools", {}), { stopReason: "toolUse" });
|
||||
},
|
||||
(context) => {
|
||||
providerPrompts.push(getCurrentSystemPrompt(context.messages));
|
||||
sessionPrompts.push(harness.session.systemPrompt);
|
||||
return fauxAssistantMessage("done");
|
||||
},
|
||||
]);
|
||||
|
||||
await harness.session.prompt("start");
|
||||
|
||||
expect(harness.session.getActiveToolNames()).toEqual(["load_more_tools", "after_load"]);
|
||||
expect(addedToolNames).toEqual([["after_load"]]);
|
||||
expect(providerPrompts).toHaveLength(2);
|
||||
expect(providerPrompts[0]).not.toBe(providerPrompts[1]);
|
||||
expect(sessionPrompts).toEqual(providerPrompts);
|
||||
} finally {
|
||||
harness.cleanup();
|
||||
}
|
||||
@@ -129,32 +106,7 @@ describe("extension active tools next-turn refresh", () => {
|
||||
systemPrompt: `${event.systemPrompt}\n\nkeep this run override`,
|
||||
}));
|
||||
|
||||
pi.registerTool({
|
||||
name: "switch_tools",
|
||||
label: "Switch Tools",
|
||||
description: "Switch the active extension tool set",
|
||||
promptSnippet: "Switch to the next extension tool",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => {
|
||||
pi.setActiveTools(["after_switch"]);
|
||||
return {
|
||||
content: [{ type: "text", text: "switched" }],
|
||||
details: {},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
pi.registerTool({
|
||||
name: "after_switch",
|
||||
label: "After Switch",
|
||||
description: "Tool that should be available after switching",
|
||||
promptSnippet: "Run after the active tool set changes",
|
||||
parameters: Type.Object({}),
|
||||
execute: async () => ({
|
||||
content: [{ type: "text", text: "after" }],
|
||||
details: {},
|
||||
}),
|
||||
});
|
||||
registerSwitchTools(pi);
|
||||
},
|
||||
];
|
||||
const harness = await createHarness({
|
||||
@@ -166,15 +118,18 @@ describe("extension active tools next-turn refresh", () => {
|
||||
|
||||
const providerSystemPrompts: string[] = [];
|
||||
const providerToolNames: string[][] = [];
|
||||
const captureSystemPrompt = (context: TranscriptContext): void => {
|
||||
providerSystemPrompts.push(getCurrentSystemPrompt(context.messages));
|
||||
};
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
providerSystemPrompts.push(context.systemPrompt ?? "");
|
||||
providerToolNames.push((context.tools ?? []).map((tool) => tool.name).sort());
|
||||
captureSystemPrompt(context);
|
||||
providerToolNames.push(getProviderToolNames(context));
|
||||
return fauxAssistantMessage(fauxToolCall("switch_tools", {}), { stopReason: "toolUse" });
|
||||
},
|
||||
(context) => {
|
||||
providerSystemPrompts.push(context.systemPrompt ?? "");
|
||||
providerToolNames.push((context.tools ?? []).map((tool) => tool.name).sort());
|
||||
captureSystemPrompt(context);
|
||||
providerToolNames.push(getProviderToolNames(context));
|
||||
return fauxAssistantMessage("done");
|
||||
},
|
||||
]);
|
||||
|
||||
@@ -56,7 +56,7 @@ describe("regression #6363: agent settled event and idle waiting", () => {
|
||||
await harness.session.prompt("test");
|
||||
|
||||
expect(harness.eventsOfType("agent_end").map((event) => event.willRetry)).toEqual([true, false]);
|
||||
expect(harness.eventsOfType("agent_settled")).toHaveLength(1);
|
||||
expect(harness.eventsOfType("agent_settled")).toEqual([{ type: "agent_settled", aborted: false }]);
|
||||
expect(extensionEvents).toEqual(["agent_end", "agent_end", "agent_settled:true"]);
|
||||
expect(publicEvents).toEqual(["agent_settled"]);
|
||||
});
|
||||
|
||||
@@ -99,7 +99,7 @@ describe("#6647 compaction retries transient summarization failures", () => {
|
||||
const result = await harness.session.compact();
|
||||
|
||||
expect(result.summary).toContain("recovered summary");
|
||||
expect(getCallCount()).toBe(3); // 1 initial + 2 retries
|
||||
expect(getCallCount()).toBe(3); // 1 prefix-summary attempt + 2 retries
|
||||
const starts = harness.eventsOfType("summarization_retry_scheduled");
|
||||
const ends = harness.eventsOfType("summarization_retry_finished");
|
||||
expect(starts).toHaveLength(2);
|
||||
|
||||
@@ -1,10 +1,19 @@
|
||||
import type { Api, Model, Provider } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { AuthStorage } from "../../../src/core/auth-storage.ts";
|
||||
import { defaultModelPerProvider } from "../../../src/core/model-resolver.ts";
|
||||
import { ModelRuntime } from "../../../src/core/model-runtime.ts";
|
||||
import { InteractiveMode } from "../../../src/modes/interactive/interactive-mode.ts";
|
||||
import { createHarness, type Harness } from "../harness.ts";
|
||||
|
||||
const complete = Reflect.get(InteractiveMode.prototype, "completeProviderAuthentication") as (
|
||||
this: object,
|
||||
providerId: string,
|
||||
providerName: string,
|
||||
authType: "oauth" | "api_key",
|
||||
previousModel: Model<Api>,
|
||||
) => Promise<void>;
|
||||
|
||||
const dynamicModel: Model<"openai-completions"> = {
|
||||
id: "dynamic",
|
||||
name: "Dynamic",
|
||||
@@ -99,17 +108,8 @@ describe("issues #7027 and #7113 credential refresh hang", () => {
|
||||
showError: vi.fn(),
|
||||
showWarning,
|
||||
maybeWarnAboutAnthropicSubscriptionAuth: vi.fn(),
|
||||
checkDaxnutsEasterEgg: vi.fn(),
|
||||
ui: { requestRender: vi.fn() },
|
||||
};
|
||||
const complete = Reflect.get(InteractiveMode.prototype, "completeProviderAuthentication") as (
|
||||
this: object,
|
||||
providerId: string,
|
||||
providerName: string,
|
||||
authType: "oauth" | "api_key",
|
||||
previousModel: Model<Api>,
|
||||
) => Promise<void>;
|
||||
|
||||
await complete.call(context, dynamicModel.provider, "Stalled Login", "api_key", harness.getModel());
|
||||
expect(runtime.refresh).toHaveBeenCalledWith({
|
||||
providers: [dynamicModel.provider],
|
||||
@@ -123,3 +123,95 @@ describe("issues #7027 and #7113 credential refresh hang", () => {
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
describe("post-login model discovery", () => {
|
||||
let harness: Harness | undefined;
|
||||
afterEach(() => {
|
||||
vi.useRealTimers();
|
||||
harness?.cleanup();
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
async function startLogin() {
|
||||
harness = await createHarness();
|
||||
vi.useFakeTimers();
|
||||
const session = harness.session;
|
||||
const model = harness.getModel();
|
||||
const unknownModel = { ...model, id: "unknown", provider: "unknown", api: "unknown" };
|
||||
const currentModel = vi.spyOn(session, "model", "get").mockReturnValue(unknownModel);
|
||||
const availableModels = vi.spyOn(session.modelRuntime, "getAvailableSnapshot").mockReturnValue([]);
|
||||
let finishRefresh = () => {};
|
||||
vi.spyOn(session.modelRuntime, "refresh").mockImplementation(
|
||||
(options) =>
|
||||
new Promise((resolve) => {
|
||||
finishRefresh = () => resolve({ aborted: false, errors: new Map() });
|
||||
options?.signal?.addEventListener("abort", () => resolve({ aborted: true, errors: new Map() }), {
|
||||
once: true,
|
||||
});
|
||||
}),
|
||||
);
|
||||
const setModel = vi.spyOn(session, "setModel").mockResolvedValue();
|
||||
const context = {
|
||||
session,
|
||||
updateAvailableProviderCount: vi.fn(),
|
||||
footer: { invalidate: vi.fn() },
|
||||
updateEditorBorderColor: vi.fn(),
|
||||
showStatus: vi.fn(),
|
||||
showError: vi.fn(),
|
||||
showWarning: vi.fn(),
|
||||
maybeWarnAboutAnthropicSubscriptionAuth: vi.fn(),
|
||||
ui: { requestRender: vi.fn() },
|
||||
};
|
||||
await complete.call(context, "radius", "Radius", "oauth", unknownModel);
|
||||
expect(context.showStatus).toHaveBeenCalledWith(expect.stringContaining("Credentials saved"));
|
||||
expect(context.showError).not.toHaveBeenCalled();
|
||||
expect(setModel).not.toHaveBeenCalled();
|
||||
|
||||
return {
|
||||
...context,
|
||||
setModel,
|
||||
currentModel,
|
||||
async discover(ids: string[]) {
|
||||
availableModels.mockReturnValue(ids.map((id) => ({ ...model, provider: "radius", id })));
|
||||
finishRefresh();
|
||||
await vi.advanceTimersByTimeAsync(0);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
it.each([
|
||||
{ models: ["fast", "balanced"], selected: "balanced" },
|
||||
{ models: ["fast", "powerful"], selected: "fast" },
|
||||
])("selects $selected from the refreshed catalog $models", async ({ models, selected }) => {
|
||||
expect(defaultModelPerProvider.radius).toBe("balanced");
|
||||
const login = await startLogin();
|
||||
await login.discover(models);
|
||||
expect(login.setModel).toHaveBeenCalledWith(expect.objectContaining({ provider: "radius", id: selected }), {
|
||||
persist: true,
|
||||
});
|
||||
expect(login.showError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("reports an empty catalog only after refresh", async () => {
|
||||
const login = await startLogin();
|
||||
await login.discover([]);
|
||||
expect(login.setModel).not.toHaveBeenCalled();
|
||||
expect(login.showError).toHaveBeenCalledWith(expect.stringContaining("no models are available"));
|
||||
});
|
||||
|
||||
it("preserves a model selected during refresh", async () => {
|
||||
const login = await startLogin();
|
||||
login.currentModel.mockReturnValue(harness!.getModel());
|
||||
await login.discover(["fast", "balanced"]);
|
||||
expect(login.setModel).not.toHaveBeenCalled();
|
||||
expect(login.showError).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it("bounds refresh to 15 seconds", async () => {
|
||||
const login = await startLogin();
|
||||
await vi.advanceTimersByTimeAsync(15_000);
|
||||
expect(login.showWarning).toHaveBeenCalledWith(expect.stringContaining("timed out"));
|
||||
expect(login.showError).toHaveBeenCalledWith(expect.stringContaining("no models are available"));
|
||||
expect(login.setModel).not.toHaveBeenCalled();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { fauxAssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { PromptDisposition } from "../../../src/core/agent-session.ts";
|
||||
import { createHarness, getMessageText, getUserTexts, type Harness } from "../harness.ts";
|
||||
|
||||
describe("issue #7150: RPC prompt during manual compaction", () => {
|
||||
@@ -57,13 +58,13 @@ describe("issue #7150: RPC prompt during manual compaction", () => {
|
||||
const compactPromise = harness.session.compact();
|
||||
await compactionStarted;
|
||||
|
||||
let preflightResult: boolean | undefined;
|
||||
let preflightResult: PromptDisposition | undefined;
|
||||
let promptError: unknown;
|
||||
try {
|
||||
await harness.session.prompt("PROBE-7150", {
|
||||
source: "rpc",
|
||||
preflightResult: (success) => {
|
||||
preflightResult = success;
|
||||
preflightResult: (result) => {
|
||||
preflightResult = result;
|
||||
},
|
||||
});
|
||||
} catch (error) {
|
||||
@@ -79,7 +80,7 @@ describe("issue #7150: RPC prompt during manual compaction", () => {
|
||||
entry.type === "message" && entry.message.role === "user" ? [getMessageText(entry.message)] : [],
|
||||
);
|
||||
|
||||
expect(preflightResult).toBe(false);
|
||||
expect(preflightResult).toBeUndefined();
|
||||
expect(promptError).toEqual(
|
||||
expect.objectContaining({ message: expect.stringContaining("compaction is in progress") }),
|
||||
);
|
||||
|
||||
@@ -9,13 +9,28 @@ const state = vi.hoisted(() => {
|
||||
});
|
||||
return {
|
||||
originalGetBuiltinModule,
|
||||
jitiModuleLoads: 0,
|
||||
virtualModulesLoads: 0,
|
||||
createJiti: vi.fn((_id: unknown, _options: unknown) => ({
|
||||
import: vi.fn(async () => () => {}),
|
||||
})),
|
||||
};
|
||||
});
|
||||
|
||||
vi.mock("jiti/static", () => ({ createJiti: state.createJiti }));
|
||||
vi.mock("jiti/static", () => {
|
||||
state.jitiModuleLoads++;
|
||||
return { createJiti: state.createJiti };
|
||||
});
|
||||
|
||||
vi.mock("../../../src/core/extensions/virtual-modules.ts", () => {
|
||||
state.virtualModulesLoads++;
|
||||
return {
|
||||
VIRTUAL_MODULES: {
|
||||
typebox: {},
|
||||
"@earendil-works/pi-coding-agent": {},
|
||||
},
|
||||
};
|
||||
});
|
||||
|
||||
import { loadExtensions } from "../../../src/core/extensions/loader.ts";
|
||||
|
||||
@@ -32,10 +47,16 @@ afterAll(() => {
|
||||
});
|
||||
|
||||
describe("Node SEA extension loading", () => {
|
||||
it("uses bundled virtual modules instead of filesystem aliases", async () => {
|
||||
// Regression test for #8237 and #9540.
|
||||
it("loads jiti and bundled virtual modules only when importing an extension", async () => {
|
||||
expect(state.jitiModuleLoads).toBe(0);
|
||||
expect(state.virtualModulesLoads).toBe(0);
|
||||
|
||||
const result = await loadExtensions(["/extension.ts"], "/");
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(state.jitiModuleLoads).toBe(1);
|
||||
expect(state.virtualModulesLoads).toBe(1);
|
||||
expect(result.extensions).toHaveLength(1);
|
||||
expect(state.createJiti).toHaveBeenCalledOnce();
|
||||
|
||||
|
||||
@@ -48,7 +48,14 @@ describe("#8537 custom messages injected during tool execution", () => {
|
||||
|
||||
await harness.session.prompt("hi");
|
||||
|
||||
expect(roles(harness.session.messages)).toEqual(["user", "assistant", "toolResult", "custom", "assistant"]);
|
||||
expect(roles(harness.session.messages)).toEqual([
|
||||
"system",
|
||||
"user",
|
||||
"assistant",
|
||||
"toolResult",
|
||||
"custom",
|
||||
"assistant",
|
||||
]);
|
||||
});
|
||||
|
||||
it("keeps session entries and message events in the same order as agent state", async () => {
|
||||
@@ -84,13 +91,13 @@ describe("#8537 custom messages injected during tool execution", () => {
|
||||
.flatMap((entry) =>
|
||||
entry.type === "message" ? [entry.message.role] : entry.type === "custom_message" ? ["custom"] : [],
|
||||
);
|
||||
expect(entryKinds).toEqual(["user", "assistant", "toolResult", "custom", "assistant"]);
|
||||
expect(entryKinds).toEqual(["system", "user", "assistant", "toolResult", "custom", "assistant"]);
|
||||
|
||||
// message events must never describe a message the session tree does not contain yet
|
||||
const messageStarts = harness.events.flatMap((event) =>
|
||||
event.type === "message_start" ? [event.message.role] : [],
|
||||
);
|
||||
expect(messageStarts).toEqual(["user", "assistant", "toolResult", "custom", "assistant"]);
|
||||
expect(messageStarts).toEqual(["system", "user", "assistant", "toolResult", "custom", "assistant"]);
|
||||
});
|
||||
|
||||
it("produces an llm history where every tool result follows its tool call", async () => {
|
||||
|
||||
@@ -83,8 +83,13 @@ describe("regression #8724: in-memory fork during an active tool turn", () => {
|
||||
await runtime.session.bindExtensions({});
|
||||
|
||||
expect(forkResult).toEqual({ cancelled: false, selectedText: "first prompt" });
|
||||
expect(runtime.session.messages).toEqual([]);
|
||||
expect(runtime.session.sessionManager.getEntries().filter((entry) => entry.type === "message")).toEqual([]);
|
||||
expect(runtime.session.messages.map((message) => message.role)).toEqual(["system"]);
|
||||
expect(
|
||||
runtime.session.sessionManager
|
||||
.getEntries()
|
||||
.filter((entry) => entry.type === "message")
|
||||
.map((entry) => entry.message.role),
|
||||
).toEqual(["system"]);
|
||||
|
||||
let capturedRoles: string[] = [];
|
||||
harness.setResponses([
|
||||
@@ -95,6 +100,6 @@ describe("regression #8724: in-memory fork during an active tool turn", () => {
|
||||
]);
|
||||
await runtime.session.prompt("next prompt");
|
||||
|
||||
expect(capturedRoles).toEqual(["user"]);
|
||||
expect(capturedRoles).toEqual(["system", "system", "user"]);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -0,0 +1,63 @@
|
||||
import { type AssistantMessage, fauxAssistantMessage, fauxProvider } from "@earendil-works/pi-ai";
|
||||
import { getApiProvider } from "@earendil-works/pi-ai/compat";
|
||||
import { expect, it } from "vitest";
|
||||
import { createHarness } from "../harness.ts";
|
||||
|
||||
// Regression for #8964: extensions can stream responses from providers registered with pi.registerProvider().
|
||||
it.each(["stream", "streamSimple"] as const)(
|
||||
"allows an extension command to use ctx.modelRegistry.%s",
|
||||
async (method) => {
|
||||
const faux = fauxProvider({ provider: "extension-provider", api: "issue-8964-extension-api" });
|
||||
let receivedApiKey: string | undefined;
|
||||
faux.setResponses([
|
||||
(_context, options) => {
|
||||
receivedApiKey = options?.apiKey;
|
||||
return fauxAssistantMessage("custom provider response");
|
||||
},
|
||||
]);
|
||||
let streamedText = "";
|
||||
let result: AssistantMessage | undefined;
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.registerProvider(faux.provider.id, {
|
||||
api: faux.api,
|
||||
baseUrl: faux.getModel().baseUrl,
|
||||
apiKey: "extension-key",
|
||||
models: faux.models,
|
||||
streamSimple: faux.provider.streamSimple,
|
||||
});
|
||||
},
|
||||
(pi) => {
|
||||
pi.registerCommand("stream-custom", {
|
||||
description: "Stream a response from the custom provider",
|
||||
handler: async (_args, ctx) => {
|
||||
const model = ctx.modelRegistry.find(faux.provider.id, faux.getModel().id)!;
|
||||
const stream = ctx.modelRegistry[method](model, {
|
||||
messages: [{ role: "user", content: "Hello", timestamp: Date.now() }],
|
||||
});
|
||||
for await (const event of stream) {
|
||||
if (event.type === "text_delta") streamedText += event.delta;
|
||||
}
|
||||
result = await stream.result();
|
||||
},
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
try {
|
||||
expect(getApiProvider(faux.api)).toBeUndefined();
|
||||
await harness.session.prompt("/stream-custom");
|
||||
|
||||
expect(receivedApiKey).toBe("extension-key");
|
||||
expect(streamedText).toBe("custom provider response");
|
||||
expect(result).toMatchObject({
|
||||
stopReason: "stop",
|
||||
content: [{ type: "text", text: "custom provider response" }],
|
||||
});
|
||||
expect(getApiProvider(faux.api)).toBeUndefined();
|
||||
} finally {
|
||||
harness.cleanup();
|
||||
}
|
||||
},
|
||||
);
|
||||
272
third_party/pi/packages/coding-agent/test/suite/regressions/9068-user-bash-fail-closed.test.ts
vendored
Normal file
272
third_party/pi/packages/coding-agent/test/suite/regressions/9068-user-bash-fail-closed.test.ts
vendored
Normal file
@@ -0,0 +1,272 @@
|
||||
import { afterEach, describe, expect, test, vi } from "vitest";
|
||||
import type { AgentSessionRuntime } from "../../../src/core/agent-session-runtime.ts";
|
||||
import type { ExtensionAPI, UserBashEvent, UserBashEventResult } from "../../../src/core/extensions/types.ts";
|
||||
import { InteractiveMode } from "../../../src/modes/interactive/interactive-mode.ts";
|
||||
import { runRpcMode } from "../../../src/modes/rpc/rpc-mode.ts";
|
||||
import { createHarness, type Harness } from "../harness.ts";
|
||||
|
||||
// Regression for https://github.com/earendil-works/pi/issues/9068
|
||||
|
||||
const rpcIo = vi.hoisted(() => ({
|
||||
outputLines: [] as string[],
|
||||
lineHandler: undefined as ((line: string) => void) | undefined,
|
||||
}));
|
||||
|
||||
vi.mock("../../../src/core/output-guard.js", () => ({
|
||||
flushRawStdout: vi.fn(async () => {}),
|
||||
takeOverStdout: vi.fn(),
|
||||
waitForRawStdoutBackpressure: vi.fn(async () => {}),
|
||||
writeRawStdout: (line: string) => {
|
||||
rpcIo.outputLines.push(line);
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../../../src/modes/interactive/theme/theme.js", () => ({ theme: {} }));
|
||||
|
||||
vi.mock("../../../src/modes/interactive/components/bash-execution.js", () => ({
|
||||
BashExecutionComponent: class {
|
||||
appendOutput(): void {}
|
||||
setComplete(): void {}
|
||||
},
|
||||
}));
|
||||
|
||||
vi.mock("../../../src/modes/rpc/jsonl.js", () => ({
|
||||
attachJsonlLineReader: vi.fn((_stream: NodeJS.ReadableStream, onLine: (line: string) => void) => {
|
||||
rpcIo.lineHandler = onLine;
|
||||
return () => {
|
||||
rpcIo.lineHandler = undefined;
|
||||
};
|
||||
}),
|
||||
serializeJsonLine: (value: unknown) => `${JSON.stringify(value)}\n`,
|
||||
}));
|
||||
|
||||
type NodeListener = Parameters<typeof process.on>[1];
|
||||
|
||||
type ListenerSnapshot = {
|
||||
stdinEnd: NodeListener[];
|
||||
signals: Map<NodeJS.Signals, NodeListener[]>;
|
||||
};
|
||||
|
||||
function takeListenerSnapshot(): ListenerSnapshot {
|
||||
const signals: NodeJS.Signals[] = process.platform === "win32" ? ["SIGTERM"] : ["SIGTERM", "SIGHUP"];
|
||||
return {
|
||||
stdinEnd: process.stdin.listeners("end") as NodeListener[],
|
||||
signals: new Map(signals.map((signal) => [signal, process.listeners(signal) as NodeListener[]])),
|
||||
};
|
||||
}
|
||||
|
||||
function restoreListeners(snapshot: ListenerSnapshot): void {
|
||||
for (const listener of process.stdin.listeners("end") as NodeListener[]) {
|
||||
if (!snapshot.stdinEnd.includes(listener)) process.stdin.off("end", listener);
|
||||
}
|
||||
for (const [signal, previousListeners] of snapshot.signals) {
|
||||
for (const listener of process.listeners(signal) as NodeListener[]) {
|
||||
if (!previousListeners.includes(listener)) process.off(signal, listener);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function parseOutputLines(): Array<Record<string, unknown>> {
|
||||
return rpcIo.outputLines
|
||||
.flatMap((line) => line.split("\n"))
|
||||
.filter((line) => line.trim().length > 0)
|
||||
.map((line) => JSON.parse(line) as Record<string, unknown>);
|
||||
}
|
||||
|
||||
function createRuntimeHost(harness: Harness): AgentSessionRuntime {
|
||||
return {
|
||||
session: harness.session,
|
||||
newSession: vi.fn(async () => ({ cancelled: true })),
|
||||
switchSession: vi.fn(async () => ({ cancelled: true })),
|
||||
fork: vi.fn(async () => ({ cancelled: true, selectedText: "" })),
|
||||
dispose: vi.fn(async () => {}),
|
||||
setRebindSession: vi.fn(),
|
||||
} as unknown as AgentSessionRuntime;
|
||||
}
|
||||
|
||||
async function startRpcHarness(extension: (pi: ExtensionAPI) => void): Promise<{
|
||||
harness: Harness;
|
||||
send(command: Record<string, unknown>): void;
|
||||
cleanup(): void;
|
||||
}> {
|
||||
const listenerSnapshot = takeListenerSnapshot();
|
||||
const harness = await createHarness({ extensionFactories: [extension] });
|
||||
const cleanup = () => {
|
||||
harness.cleanup();
|
||||
restoreListeners(listenerSnapshot);
|
||||
};
|
||||
|
||||
try {
|
||||
void runRpcMode(createRuntimeHost(harness));
|
||||
await vi.waitFor(() => expect(rpcIo.lineHandler).toBeDefined());
|
||||
} catch (error) {
|
||||
cleanup();
|
||||
throw error;
|
||||
}
|
||||
|
||||
return {
|
||||
harness,
|
||||
send(command) {
|
||||
if (!rpcIo.lineHandler) throw new Error("RPC line handler is not attached");
|
||||
rpcIo.lineHandler(JSON.stringify(command));
|
||||
},
|
||||
cleanup,
|
||||
};
|
||||
}
|
||||
|
||||
type InteractiveBashContext = {
|
||||
defaultEditor: { onSubmit?: (text: string) => Promise<void> | void };
|
||||
editor: { addToHistory?: (text: string) => void };
|
||||
session: Harness["session"];
|
||||
sessionManager: Harness["sessionManager"];
|
||||
ui: { requestRender(): void };
|
||||
chatContainer: { addChild(component: unknown): void };
|
||||
pendingMessagesContainer: { addChild(component: unknown): void };
|
||||
pendingBashComponents: unknown[];
|
||||
isBashMode: boolean;
|
||||
handleBashCommand(command: string, excludeFromContext?: boolean): Promise<void>;
|
||||
showError(message: string): void;
|
||||
updateEditorBorderColor(): void;
|
||||
};
|
||||
|
||||
const interactiveModePrototype = InteractiveMode.prototype as unknown as {
|
||||
setupEditorSubmitHandler(this: InteractiveBashContext): void;
|
||||
handleBashCommand(this: InteractiveBashContext, command: string, excludeFromContext?: boolean): Promise<void>;
|
||||
};
|
||||
|
||||
const localResult = {
|
||||
output: "local output",
|
||||
exitCode: 0,
|
||||
cancelled: false,
|
||||
truncated: false,
|
||||
};
|
||||
|
||||
const rpcCases: Array<{
|
||||
name: string;
|
||||
extension: (pi: ExtensionAPI) => void;
|
||||
error?: string;
|
||||
executeCount: number;
|
||||
}> = [
|
||||
{
|
||||
name: "fails the request without executing bash when a handler throws",
|
||||
extension: (pi) => {
|
||||
pi.on("user_bash", async () => {
|
||||
throw new Error("Routing failed");
|
||||
});
|
||||
},
|
||||
error: "Routing failed",
|
||||
executeCount: 0,
|
||||
},
|
||||
{
|
||||
name: "fails the request without executing bash when a handler returns an empty result",
|
||||
extension: (pi) => {
|
||||
pi.on("user_bash", async () => ({}) as unknown as UserBashEventResult);
|
||||
},
|
||||
error: "Invalid user_bash handler result",
|
||||
executeCount: 0,
|
||||
},
|
||||
{
|
||||
name: "executes bash normally when a handler returns undefined",
|
||||
extension: (pi) => {
|
||||
pi.on("user_bash", async () => undefined);
|
||||
},
|
||||
executeCount: 1,
|
||||
},
|
||||
];
|
||||
|
||||
afterEach(() => {
|
||||
rpcIo.outputLines = [];
|
||||
rpcIo.lineHandler = undefined;
|
||||
});
|
||||
|
||||
describe("RPC user_bash failure handling (#9068)", () => {
|
||||
test.each(rpcCases)("$name", async ({ extension, error, executeCount }) => {
|
||||
const rpc = await startRpcHarness(extension);
|
||||
const executeBash = vi.spyOn(rpc.harness.session, "executeBash").mockResolvedValue(localResult);
|
||||
|
||||
try {
|
||||
rpc.send({ id: "bash-request", type: "bash", command: "pwd" });
|
||||
|
||||
await vi.waitFor(() => {
|
||||
const response = parseOutputLines().find(
|
||||
(output) => output.type === "response" && output.id === "bash-request",
|
||||
);
|
||||
expect(response).toMatchObject({
|
||||
id: "bash-request",
|
||||
type: "response",
|
||||
command: "bash",
|
||||
success: error === undefined,
|
||||
...(error ? { error: expect.stringContaining(error) } : { data: localResult }),
|
||||
});
|
||||
});
|
||||
|
||||
if (error) {
|
||||
expect(parseOutputLines()).toEqual(
|
||||
expect.arrayContaining([
|
||||
expect.objectContaining({
|
||||
type: "extension_error",
|
||||
event: "user_bash",
|
||||
error: expect.stringContaining(error),
|
||||
}),
|
||||
]),
|
||||
);
|
||||
}
|
||||
expect(executeBash).toHaveBeenCalledTimes(executeCount);
|
||||
} finally {
|
||||
executeBash.mockRestore();
|
||||
rpc.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("Interactive user_bash failure handling (#9068)", () => {
|
||||
test.each([
|
||||
["!pwd", false],
|
||||
["!!pwd", true],
|
||||
])("fails closed for %s when a handler returns an empty result", async (input, excludeFromContext) => {
|
||||
const events: UserBashEvent[] = [];
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("user_bash", async (event) => {
|
||||
events.push(event);
|
||||
return {} as unknown as UserBashEventResult;
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
const executeBash = vi.spyOn(harness.session, "executeBash").mockResolvedValue(localResult);
|
||||
const context: InteractiveBashContext = {
|
||||
defaultEditor: {},
|
||||
editor: { addToHistory: vi.fn() },
|
||||
session: harness.session,
|
||||
sessionManager: harness.sessionManager,
|
||||
ui: { requestRender: vi.fn() },
|
||||
chatContainer: { addChild: vi.fn() },
|
||||
pendingMessagesContainer: { addChild: vi.fn() },
|
||||
pendingBashComponents: [],
|
||||
isBashMode: true,
|
||||
handleBashCommand: interactiveModePrototype.handleBashCommand,
|
||||
showError: vi.fn(),
|
||||
updateEditorBorderColor: vi.fn(),
|
||||
};
|
||||
interactiveModePrototype.setupEditorSubmitHandler.call(context);
|
||||
|
||||
try {
|
||||
await context.defaultEditor.onSubmit?.(input);
|
||||
|
||||
expect(events).toEqual([
|
||||
{
|
||||
type: "user_bash",
|
||||
command: "pwd",
|
||||
excludeFromContext,
|
||||
cwd: harness.sessionManager.getCwd(),
|
||||
},
|
||||
]);
|
||||
expect(executeBash).not.toHaveBeenCalled();
|
||||
} finally {
|
||||
executeBash.mockRestore();
|
||||
harness.cleanup();
|
||||
}
|
||||
});
|
||||
});
|
||||
102
third_party/pi/packages/coding-agent/test/suite/regressions/9178-tree-during-compaction.test.ts
vendored
Normal file
102
third_party/pi/packages/coding-agent/test/suite/regressions/9178-tree-during-compaction.test.ts
vendored
Normal file
@@ -0,0 +1,102 @@
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import { assistantMsg, userMsg } from "../../utilities.ts";
|
||||
import { createHarness, getMessageText, type Harness } from "../harness.ts";
|
||||
|
||||
function createDeferred(): { promise: Promise<void>; resolve: () => void } {
|
||||
let resolve!: () => void;
|
||||
const promise = new Promise<void>((promiseResolve) => {
|
||||
resolve = promiseResolve;
|
||||
});
|
||||
return { promise, resolve };
|
||||
}
|
||||
|
||||
describe("issue #9178: tree navigation during manual compaction", () => {
|
||||
let harness: Harness | undefined;
|
||||
|
||||
afterEach(() => harness?.cleanup());
|
||||
|
||||
it("rejects navigation before the active leaf can change", async () => {
|
||||
const compactionStarted = createDeferred();
|
||||
const compactionReleased = createDeferred();
|
||||
|
||||
harness = await createHarness({
|
||||
settings: { compaction: { keepRecentTokens: 1 } },
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", async (event) => {
|
||||
compactionStarted.resolve();
|
||||
await compactionReleased.promise;
|
||||
return {
|
||||
compaction: {
|
||||
summary: "summary",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
details: {},
|
||||
},
|
||||
};
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
harness.sessionManager.appendMessage(userMsg("first user"));
|
||||
const navigationTargetId = harness.sessionManager.appendMessage(assistantMsg("first assistant"));
|
||||
harness.sessionManager.appendMessage(userMsg("second user"));
|
||||
const originalLeafId = harness.sessionManager.appendMessage(assistantMsg("second assistant"));
|
||||
harness.session.agent.state.messages = harness.sessionManager.buildSessionContext().messages;
|
||||
|
||||
const compactionPromise = harness.session.compact();
|
||||
await compactionStarted.promise;
|
||||
|
||||
expect(harness.session.isCompacting).toBe(true);
|
||||
try {
|
||||
await expect(harness.session.navigateTree(navigationTargetId, { summarize: false })).rejects.toThrow(
|
||||
"Wait for the current compaction or tree navigation to finish before navigating the session tree.",
|
||||
);
|
||||
expect(harness.sessionManager.getLeafId()).toBe(originalLeafId);
|
||||
} finally {
|
||||
compactionReleased.resolve();
|
||||
}
|
||||
await compactionPromise;
|
||||
|
||||
expect(harness.sessionManager.getEntries().at(-1)).toMatchObject({
|
||||
type: "compaction",
|
||||
parentId: originalLeafId,
|
||||
});
|
||||
expect(harness.session.messages.map(getMessageText)).toContain("second assistant");
|
||||
});
|
||||
|
||||
it("rejects a second navigation while the first is waiting", async () => {
|
||||
const navigationStarted = createDeferred();
|
||||
const navigationReleased = createDeferred();
|
||||
harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_tree", async () => {
|
||||
navigationStarted.resolve();
|
||||
await navigationReleased.promise;
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const secondTargetId = harness.sessionManager.appendMessage(userMsg("first user"));
|
||||
const firstTargetId = harness.sessionManager.appendMessage(assistantMsg("first assistant"));
|
||||
harness.sessionManager.appendMessage(userMsg("second user"));
|
||||
const originalLeafId = harness.sessionManager.appendMessage(assistantMsg("second assistant"));
|
||||
harness.session.agent.state.messages = harness.sessionManager.buildSessionContext().messages;
|
||||
|
||||
const firstNavigation = harness.session.navigateTree(firstTargetId, { summarize: false });
|
||||
await navigationStarted.promise;
|
||||
|
||||
const secondNavigation = harness.session.navigateTree(secondTargetId, { summarize: false });
|
||||
expect(harness.sessionManager.getLeafId()).toBe(originalLeafId);
|
||||
navigationReleased.resolve();
|
||||
|
||||
await expect(secondNavigation).rejects.toThrow(
|
||||
"Wait for the current compaction or tree navigation to finish before navigating the session tree.",
|
||||
);
|
||||
await firstNavigation;
|
||||
expect(harness.sessionManager.getLeafId()).toBe(firstTargetId);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,158 @@
|
||||
import { type AssistantMessage, fauxAssistantMessage } from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import { createHarness, type Harness } from "../harness.ts";
|
||||
|
||||
type SessionWithCompactionInternals = {
|
||||
_runAutoCompaction: (reason: "overflow" | "threshold", willRetry: boolean) => Promise<boolean>;
|
||||
};
|
||||
|
||||
function seedCompactableSession(harness: Harness): void {
|
||||
const model = harness.getModel();
|
||||
harness.sessionManager.appendMessage({
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "x".repeat(500) }],
|
||||
timestamp: 1,
|
||||
});
|
||||
const assistant: AssistantMessage = {
|
||||
...fauxAssistantMessage("y".repeat(200), { timestamp: 2 }),
|
||||
api: model.api,
|
||||
provider: model.provider,
|
||||
model: model.id,
|
||||
usage: {
|
||||
input: 100,
|
||||
output: 0,
|
||||
cacheRead: 0,
|
||||
cacheWrite: 0,
|
||||
totalTokens: 100,
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 },
|
||||
},
|
||||
};
|
||||
harness.sessionManager.appendMessage(assistant);
|
||||
harness.session.agent.state.messages = harness.sessionManager.buildSessionContext().messages;
|
||||
}
|
||||
|
||||
function runAutoCompaction(harness: Harness): Promise<boolean> {
|
||||
return (harness.session as unknown as SessionWithCompactionInternals)._runAutoCompaction("threshold", false);
|
||||
}
|
||||
|
||||
describe("automatic compaction cancellation regressions", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
while (harnesses.length > 0) harnesses.pop()?.cleanup();
|
||||
});
|
||||
|
||||
// Regression test for #9340.
|
||||
it("does not start post-run auto-compaction after abort", async () => {
|
||||
const harness = await createHarness({
|
||||
models: [{ id: "faux-1", contextWindow: 200, maxTokens: 50 }],
|
||||
settings: {
|
||||
compaction: { enabled: true, reserveTokens: 50, keepRecentTokens: 1 },
|
||||
retry: { enabled: false },
|
||||
},
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", () => ({ cancel: true }));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
seedCompactableSession(harness);
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "Synthetic network failure" }),
|
||||
]);
|
||||
harness.session.subscribe((event) => {
|
||||
if (event.type === "message_end" && event.message.role === "assistant") {
|
||||
harness.session.abortCompaction();
|
||||
void harness.session.abort();
|
||||
}
|
||||
});
|
||||
|
||||
await harness.session.prompt("z".repeat(1000));
|
||||
|
||||
expect(harness.eventsOfType("compaction_start")).toHaveLength(0);
|
||||
});
|
||||
|
||||
// Regression test for #9777.
|
||||
it("cancels summarization authentication", async () => {
|
||||
const harness = await createHarness({ settings: { compaction: { keepRecentTokens: 1 } } });
|
||||
harnesses.push(harness);
|
||||
seedCompactableSession(harness);
|
||||
let markAuthStarted = () => {};
|
||||
const authStarted = new Promise<void>((resolve) => {
|
||||
markAuthStarted = resolve;
|
||||
});
|
||||
let authSignal: AbortSignal | undefined;
|
||||
vi.spyOn(harness.session.modelRuntime, "getAuth").mockImplementation(async (_model, options) => {
|
||||
authSignal = options?.signal;
|
||||
markAuthStarted();
|
||||
if (!authSignal) throw new Error("Missing auth abort signal");
|
||||
return await new Promise<never>((_resolve, reject) => {
|
||||
authSignal?.addEventListener("abort", () => reject(authSignal?.reason), { once: true });
|
||||
});
|
||||
});
|
||||
|
||||
const compaction = runAutoCompaction(harness);
|
||||
await authStarted;
|
||||
const started = harness.eventsOfType("compaction_start").length;
|
||||
const wasCompacting = harness.session.isCompacting;
|
||||
await Promise.all([compaction, harness.session.abort()]);
|
||||
|
||||
expect({ started, wasCompacting, authAborted: authSignal?.aborted }).toEqual({
|
||||
started: 1,
|
||||
wasCompacting: true,
|
||||
authAborted: true,
|
||||
});
|
||||
expect(harness.eventsOfType("compaction_end").at(-1)?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
// Regression test for #9777.
|
||||
it("cancels synchronously from compaction_start", async () => {
|
||||
const harness = await createHarness({ settings: { compaction: { keepRecentTokens: 1 } } });
|
||||
harnesses.push(harness);
|
||||
seedCompactableSession(harness);
|
||||
harness.session.subscribe((event) => {
|
||||
if (event.type === "compaction_start") harness.session.abortCompaction();
|
||||
});
|
||||
|
||||
await runAutoCompaction(harness);
|
||||
|
||||
expect(harness.faux.state.callCount).toBe(0);
|
||||
expect(harness.eventsOfType("compaction_end").at(-1)?.aborted).toBe(true);
|
||||
});
|
||||
|
||||
// Regression test for #9777.
|
||||
it.each([
|
||||
["matching error text", () => new Error("Compaction cancelled")],
|
||||
["an unrelated AbortError", () => Object.assign(new Error("auth failed"), { name: "AbortError" })],
|
||||
] as const)("reports %s as a failure", async (_label, createError) => {
|
||||
const harness = await createHarness({ settings: { compaction: { keepRecentTokens: 1 } } });
|
||||
harnesses.push(harness);
|
||||
seedCompactableSession(harness);
|
||||
vi.spyOn(harness.session.modelRuntime, "getAuth").mockRejectedValue(createError());
|
||||
|
||||
await runAutoCompaction(harness);
|
||||
|
||||
const event = harness.eventsOfType("compaction_end").at(-1);
|
||||
expect(event?.aborted).toBe(false);
|
||||
expect(event?.errorMessage).toContain(createError().message);
|
||||
});
|
||||
|
||||
it("reports extension cancellation as aborted", async () => {
|
||||
const harness = await createHarness({
|
||||
settings: { compaction: { keepRecentTokens: 1 } },
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", () => ({ cancel: true }));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
seedCompactableSession(harness);
|
||||
|
||||
await runAutoCompaction(harness);
|
||||
|
||||
expect(harness.eventsOfType("compaction_end").at(-1)?.aborted).toBe(true);
|
||||
});
|
||||
});
|
||||
58
third_party/pi/packages/coding-agent/test/suite/regressions/9540-extension-loader-lazy.test.ts
vendored
Normal file
58
third_party/pi/packages/coding-agent/test/suite/regressions/9540-extension-loader-lazy.test.ts
vendored
Normal file
@@ -0,0 +1,58 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
const state = vi.hoisted(() => ({
|
||||
jitiModuleLoads: 0,
|
||||
jitiStaticModuleLoads: 0,
|
||||
virtualModulesLoads: 0,
|
||||
createJiti: vi.fn((_id: unknown, _options: unknown) => ({
|
||||
import: vi.fn(async () => () => {}),
|
||||
})),
|
||||
}));
|
||||
|
||||
vi.mock("jiti", () => {
|
||||
state.jitiModuleLoads++;
|
||||
return { createJiti: state.createJiti };
|
||||
});
|
||||
|
||||
vi.mock("jiti/static", () => {
|
||||
state.jitiStaticModuleLoads++;
|
||||
return { createJiti: state.createJiti };
|
||||
});
|
||||
|
||||
vi.mock("../../../src/core/extensions/virtual-modules.ts", () => {
|
||||
state.virtualModulesLoads++;
|
||||
return { VIRTUAL_MODULES: {} };
|
||||
});
|
||||
|
||||
import { loadExtensions } from "../../../src/core/extensions/loader.ts";
|
||||
|
||||
interface JitiOptionsProbe {
|
||||
alias?: unknown;
|
||||
tryNative?: boolean;
|
||||
tsconfigPaths?: boolean;
|
||||
virtualModules?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
describe("extension loader lazy imports", () => {
|
||||
// Regression test for #9540.
|
||||
it("defers ordinary jiti and its virtual modules until importing an extension", async () => {
|
||||
expect(state.jitiModuleLoads).toBe(0);
|
||||
expect(state.jitiStaticModuleLoads).toBe(0);
|
||||
expect(state.virtualModulesLoads).toBe(0);
|
||||
|
||||
const result = await loadExtensions(["/extension.ts"], "/");
|
||||
|
||||
expect(result.errors).toEqual([]);
|
||||
expect(result.extensions).toHaveLength(1);
|
||||
expect(state.jitiModuleLoads).toBe(1);
|
||||
expect(state.jitiStaticModuleLoads).toBe(0);
|
||||
expect(state.virtualModulesLoads).toBe(1);
|
||||
expect(state.createJiti).toHaveBeenCalledOnce();
|
||||
|
||||
const options = state.createJiti.mock.calls[0][1] as JitiOptionsProbe;
|
||||
expect(options.tryNative).toBeUndefined();
|
||||
expect(options.tsconfigPaths).toBe(true);
|
||||
expect(options.alias).toBeUndefined();
|
||||
expect(options.virtualModules).toBeDefined();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,226 @@
|
||||
import type { AgentMessage } from "@earendil-works/pi-agent-core";
|
||||
import {
|
||||
fauxAssistantMessage,
|
||||
getCurrentSystemPrompt,
|
||||
getCurrentTools,
|
||||
type TranscriptContext,
|
||||
} from "@earendil-works/pi-ai";
|
||||
import { afterEach, describe, expect, it } from "vitest";
|
||||
import type { ExtensionFactory } from "../../../src/index.ts";
|
||||
import { createHarness, type Harness } from "../harness.ts";
|
||||
|
||||
/** Compaction supplied by an extension hook, as in the reported sessions. */
|
||||
const compactViaHook: ExtensionFactory = (pi) => {
|
||||
pi.on("session_before_compact", async (event) => ({
|
||||
compaction: {
|
||||
summary: "extension summary",
|
||||
firstKeptEntryId: event.preparation.firstKeptEntryId,
|
||||
tokensBefore: event.preparation.tokensBefore,
|
||||
details: { source: "test" },
|
||||
},
|
||||
}));
|
||||
};
|
||||
|
||||
function captureRequest(harness: Harness, text: string): () => TranscriptContext {
|
||||
let request: TranscriptContext | undefined;
|
||||
harness.setResponses([
|
||||
(context) => {
|
||||
request = context;
|
||||
return fauxAssistantMessage(text);
|
||||
},
|
||||
]);
|
||||
return () => {
|
||||
if (!request) throw new Error("expected a provider request");
|
||||
return request;
|
||||
};
|
||||
}
|
||||
|
||||
function toolNames(context: TranscriptContext): string[] {
|
||||
return getCurrentTools(context.messages).map((tool) => tool.name);
|
||||
}
|
||||
|
||||
async function compactSession(harness: Harness): Promise<void> {
|
||||
harness.settingsManager.applyOverrides({ compaction: { keepRecentTokens: 1 } });
|
||||
harness.setResponses([fauxAssistantMessage("one"), fauxAssistantMessage("two")]);
|
||||
await harness.session.prompt("first");
|
||||
await harness.session.prompt("second");
|
||||
await harness.session.compact();
|
||||
expect(harness.session.messages.map((message) => message.role).slice(0, 2)).toEqual(["system", "compactionSummary"]);
|
||||
}
|
||||
|
||||
describe("context handlers and system messages", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (harnesses.length > 0) harnesses.pop()?.cleanup();
|
||||
});
|
||||
|
||||
// Regression #9789, #9822: pruning from the compaction summary dropped the prompt and tool checkpoint.
|
||||
it("keeps the prompt and tools when a handler slices from the compaction summary", async () => {
|
||||
const seen: AgentMessage[][] = [];
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
compactViaHook,
|
||||
(pi) => {
|
||||
pi.on("context", async (event) => {
|
||||
seen.push(event.messages);
|
||||
const summary = event.messages.findIndex((message) => message.role === "compactionSummary");
|
||||
return { messages: event.messages.slice(summary) };
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
await compactSession(harness);
|
||||
const getRequest = captureRequest(harness, "after compaction");
|
||||
|
||||
await harness.session.prompt("third");
|
||||
|
||||
const request = getRequest();
|
||||
expect(seen.at(-1)?.some((message) => message.role === "system")).toBe(false);
|
||||
expect(request.messages[0]?.role).toBe("system");
|
||||
expect(toolNames(request)).toEqual(harness.session.getActiveToolNames());
|
||||
expect(getCurrentSystemPrompt(request.messages)).toBe(harness.session.systemPrompt);
|
||||
expect(request.messages.filter((message) => message.role === "system")).toHaveLength(1);
|
||||
});
|
||||
|
||||
it("keeps mid-conversation system messages in place when a handler leaves the conversation unchanged", async () => {
|
||||
let turn = 0;
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", (event) => {
|
||||
if (++turn === 2) event.systemPromptOptions.sections.plan_mode = "Plan only.";
|
||||
});
|
||||
pi.on("context", async (event) => ({ messages: event.messages }));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
harness.setResponses([fauxAssistantMessage("one")]);
|
||||
await harness.session.prompt("first");
|
||||
const getRequest = captureRequest(harness, "two");
|
||||
|
||||
await harness.session.prompt("second");
|
||||
|
||||
const systemMessages = getRequest().messages.filter((message) => message.role === "system");
|
||||
expect(systemMessages).toHaveLength(2);
|
||||
expect(systemMessages[1]?.sections).toEqual({ plan_mode: "<plan_mode>\nPlan only.\n</plan_mode>" });
|
||||
});
|
||||
|
||||
it("applies in-place edits to event.messages without a return value", async () => {
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("context", async (event) => {
|
||||
event.messages.splice(0, 0, {
|
||||
role: "user",
|
||||
content: [{ type: "text", text: "injected" }],
|
||||
timestamp: 0,
|
||||
});
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const getRequest = captureRequest(harness, "done");
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
const request = getRequest();
|
||||
expect(request.messages.map((message) => message.role)).toEqual(["system", "user", "user"]);
|
||||
expect(toolNames(request)).toEqual(harness.session.getActiveToolNames());
|
||||
});
|
||||
|
||||
it("keeps system messages a handler adds after the replayed head", async () => {
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("context", async (event) => ({
|
||||
messages: [{ role: "system", content: "ephemeral reminder", timestamp: 0 }, ...event.messages],
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const getRequest = captureRequest(harness, "done");
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
const request = getRequest();
|
||||
expect(request.messages.map((message) => message.role)).toEqual(["system", "system", "user"]);
|
||||
expect(toolNames(request)).toEqual(harness.session.getActiveToolNames());
|
||||
expect(getCurrentSystemPrompt(request.messages)).toContain(harness.session.systemPrompt);
|
||||
expect(getCurrentSystemPrompt(request.messages)).toContain("ephemeral reminder");
|
||||
});
|
||||
});
|
||||
|
||||
describe("context_with_system handlers", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
while (harnesses.length > 0) harnesses.pop()?.cleanup();
|
||||
});
|
||||
|
||||
it("runs after context handlers on the restored transcript and sends its output verbatim", async () => {
|
||||
const seen: AgentMessage[][] = [];
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
compactViaHook,
|
||||
(pi) => {
|
||||
pi.on("context_with_system", async (event) => {
|
||||
seen.push(event.messages);
|
||||
return {
|
||||
messages: event.messages.map((message) =>
|
||||
message.role === "system" && message.toolsAdded
|
||||
? { ...message, toolsAdded: message.toolsAdded.filter((tool) => tool.name !== "bash") }
|
||||
: message,
|
||||
),
|
||||
};
|
||||
});
|
||||
// Registered after, but runs first: context handlers precede context_with_system.
|
||||
pi.on("context", async (event) => {
|
||||
const summary = event.messages.findIndex((message) => message.role === "compactionSummary");
|
||||
return { messages: event.messages.slice(summary) };
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
await compactSession(harness);
|
||||
const getRequest = captureRequest(harness, "after compaction");
|
||||
|
||||
await harness.session.prompt("third");
|
||||
|
||||
const input = seen.at(-1);
|
||||
expect(input?.[0]?.role).toBe("system");
|
||||
expect(input?.[1]?.role).toBe("compactionSummary");
|
||||
expect(harness.session.getActiveToolNames()).toContain("bash");
|
||||
expect(toolNames(getRequest())).toEqual(harness.session.getActiveToolNames().filter((name) => name !== "bash"));
|
||||
});
|
||||
|
||||
it("reports a handler that drops the leading system message but honors its output", async () => {
|
||||
const harness = await createHarness({
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("context_with_system", async (event) => ({
|
||||
messages: event.messages.filter((message) => message.role !== "system"),
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
const errors: string[] = [];
|
||||
harness.session.extensionRunner.onError((error) => {
|
||||
errors.push(`${error.event}: ${error.error}`);
|
||||
});
|
||||
const getRequest = captureRequest(harness, "done");
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
expect(getRequest().messages.map((message) => message.role)).toEqual(["user"]);
|
||||
expect(errors).toEqual([
|
||||
expect.stringMatching(/^context_with_system: Handler removed the leading system message/),
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -5,6 +5,7 @@ type RebindContext = {
|
||||
session: object;
|
||||
unsubscribe?: () => void;
|
||||
applyRuntimeSettings: () => void;
|
||||
programStatus: { reset(): void };
|
||||
renderCurrentSessionState: () => void;
|
||||
bindCurrentSessionExtensions: () => Promise<void>;
|
||||
subscribeToAgent: () => void;
|
||||
@@ -40,6 +41,7 @@ describe("overlapping startup and replacement session rebinds", () => {
|
||||
const context: RebindContext = {
|
||||
session: startupSession,
|
||||
applyRuntimeSettings: () => {},
|
||||
programStatus: { reset: () => {} },
|
||||
renderCurrentSessionState: () => {},
|
||||
bindCurrentSessionExtensions: () => {
|
||||
bindCount += 1;
|
||||
|
||||
397
third_party/pi/packages/coding-agent/test/suite/virtual-models.test.ts
vendored
Normal file
397
third_party/pi/packages/coding-agent/test/suite/virtual-models.test.ts
vendored
Normal file
@@ -0,0 +1,397 @@
|
||||
import type { AgentTool } from "@earendil-works/pi-agent-core";
|
||||
import { type FauxResponseStep, fauxAssistantMessage, fauxToolCall } from "@earendil-works/pi-ai";
|
||||
import { Type } from "typebox";
|
||||
import { afterEach, describe, expect, it, vi } from "vitest";
|
||||
import type { ExtensionContext } from "../../src/core/extensions/index.ts";
|
||||
import { type ModelRoute, type ModelRouteRequest, VIRTUAL_MODEL_STATE_ENTRY } from "../../src/core/virtual-models.ts";
|
||||
import { createHarness, type Harness, type HarnessOptions } from "./harness.ts";
|
||||
|
||||
const echoTool: AgentTool = {
|
||||
name: "echo",
|
||||
label: "Echo",
|
||||
description: "Echo text back",
|
||||
parameters: Type.Object({ text: Type.String() }),
|
||||
execute: async () => ({ content: [{ type: "text", text: "echoed" }], details: {} }),
|
||||
};
|
||||
|
||||
type Route = (request: ModelRouteRequest, ctx: ExtensionContext) => ModelRoute;
|
||||
|
||||
/** Router used by these tests: new user turns pick by thinking level, everything else stays put. */
|
||||
const defaultRoute: Route = (request, ctx) => {
|
||||
const find = (id: string) => ctx.modelRegistry.find("faux", id)!;
|
||||
if (request.reason === "direct") return { model: find("large"), thinkingLevel: "low" };
|
||||
const sticky = request.failed ?? request.previous;
|
||||
if (request.reason !== "user" && sticky) {
|
||||
return { model: sticky.model, thinkingLevel: sticky.thinkingLevel ?? "high" };
|
||||
}
|
||||
return request.thinkingLevel === "high"
|
||||
? { model: find("large"), thinkingLevel: "high" }
|
||||
: { model: find("small"), thinkingLevel: "off" };
|
||||
};
|
||||
|
||||
describe("AgentSession virtual models", () => {
|
||||
const harnesses: Harness[] = [];
|
||||
|
||||
afterEach(() => {
|
||||
for (const harness of harnesses.splice(0)) harness.cleanup();
|
||||
});
|
||||
|
||||
async function createRoutedHarness(route: Route = defaultRoute, options: HarnessOptions = {}) {
|
||||
const requests: ModelRouteRequest[] = [];
|
||||
const harness = await createHarness({
|
||||
...options,
|
||||
models: [
|
||||
{ id: "small", contextWindow: 1000 },
|
||||
{ id: "large", contextWindow: 50_000, maxTokens: 4000, reasoning: true },
|
||||
],
|
||||
tools: [echoTool],
|
||||
extensionFactories: [
|
||||
...(options.extensionFactories ?? []),
|
||||
(pi) => {
|
||||
pi.registerVirtualModel({
|
||||
provider: "router",
|
||||
id: "auto",
|
||||
name: "Auto",
|
||||
thinkingLevels: ["low", "high"],
|
||||
contextWindow: 1000,
|
||||
route(request, ctx) {
|
||||
requests.push(request);
|
||||
return route(request, ctx);
|
||||
},
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harnesses.push(harness);
|
||||
// Stream through the runtime, which records the thinking level on responses.
|
||||
const runtime = harness.session.modelRuntime;
|
||||
harness.session.agent.streamFunction = (model, context, options) => runtime.streamSimple(model, context, options);
|
||||
await harness.session.setModel(runtime.getModel("router", "auto")!);
|
||||
harness.session.setThinkingLevel("high");
|
||||
const reasons = () => requests.map((request) => request.reason);
|
||||
/** Physical model and thinking level recorded on each response. */
|
||||
const dispatched = () =>
|
||||
harness.session.messages.flatMap((message) =>
|
||||
message.role === "assistant" ? [`${message.provider}/${message.model}:${message.thinkingLevel}`] : [],
|
||||
);
|
||||
return { harness, requests, reasons, dispatched };
|
||||
}
|
||||
|
||||
it("routes each request, including retries, while the selection stays virtual", async () => {
|
||||
const { harness, requests, reasons, dispatched } = await createRoutedHarness(defaultRoute, {
|
||||
settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } },
|
||||
});
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" }),
|
||||
fauxAssistantMessage(fauxToolCall("echo", { text: "hi" }), { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
expect(reasons()).toEqual(["user", "retry", "continuation"]);
|
||||
expect(requests[1].failed?.message.errorMessage).toBe("overloaded_error");
|
||||
expect(requests[2].previous?.model.id).toBe("large");
|
||||
expect(dispatched()).toEqual(["faux/large:high", "faux/large:high"]);
|
||||
expect(harness.session.model).toMatchObject({ provider: "router", id: "auto" });
|
||||
expect(harness.session.thinkingLevel).toBe("high");
|
||||
// Limits come from the physical model that produced the latest response, not the virtual model.
|
||||
expect(harness.session.getContextUsage()?.contextWindow).toBe(50_000);
|
||||
});
|
||||
|
||||
it("retries the first request of a turn on the model routed for that turn", async () => {
|
||||
const { harness, requests, reasons, dispatched } = await createRoutedHarness(defaultRoute, {
|
||||
settings: { retry: { enabled: true, maxRetries: 3, baseDelayMs: 1 } },
|
||||
});
|
||||
harness.session.setThinkingLevel("low");
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("easy answer"),
|
||||
fauxAssistantMessage("", { stopReason: "error", errorMessage: "overloaded_error" }),
|
||||
fauxAssistantMessage("hard answer"),
|
||||
]);
|
||||
await harness.session.prompt("easy");
|
||||
harness.session.setThinkingLevel("high");
|
||||
|
||||
await harness.session.prompt("hard");
|
||||
|
||||
expect(reasons()).toEqual(["user", "user", "retry"]);
|
||||
// The retry reports the failed request on large next to the small response of the previous turn.
|
||||
expect(requests[2].failed?.model.id).toBe("large");
|
||||
expect(requests[2].previous?.model.id).toBe("small");
|
||||
expect(dispatched()).toEqual(["faux/small:off", "faux/large:high"]);
|
||||
});
|
||||
|
||||
it("routes the compact-and-retry after a truncated response as a retry", async () => {
|
||||
const { harness, requests, reasons } = await createRoutedHarness(defaultRoute, {
|
||||
settings: { compaction: { keepRecentTokens: 1, reserveTokens: 0 } },
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", async ({ preparation: { firstKeptEntryId, tokensBefore } }) => ({
|
||||
compaction: { summary: "overflow compacted", firstKeptEntryId, tokensBefore },
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harness.setResponses([
|
||||
() => fauxAssistantMessage("x".repeat(64), { stopReason: "length", timestamp: Date.now() + 10_000 }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
|
||||
await harness.session.prompt("x".repeat(5000));
|
||||
|
||||
expect(harness.eventsOfType("compaction_start").map((event) => event.reason)).toEqual(["overflow"]);
|
||||
// Compaction may fold the prompt into the summary, so the retry is not a new user turn.
|
||||
expect(reasons()).toEqual(["user", "retry"]);
|
||||
expect(requests[1].failed?.model.id).toBe("large");
|
||||
expect(requests[1].failed?.message.stopReason).toBe("length");
|
||||
});
|
||||
|
||||
it("routes requests after extension messages as continuations", async () => {
|
||||
const { harness, reasons } = await createRoutedHarness(defaultRoute, {
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
let continued = false;
|
||||
pi.on("agent_before_settle", () => {
|
||||
if (continued) return undefined;
|
||||
continued = true;
|
||||
return {
|
||||
entries: [{ type: "custom_message", customType: "nudge", content: "Keep going.", display: false }],
|
||||
continue: true,
|
||||
};
|
||||
});
|
||||
},
|
||||
],
|
||||
});
|
||||
harness.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]);
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
// The hidden custom message becomes a user message for the model, but the user did not write it.
|
||||
expect(reasons()).toEqual(["user", "continuation"]);
|
||||
});
|
||||
|
||||
it("routes the first request of a prompt as a user turn when extension messages follow the prompt", async () => {
|
||||
const { harness, reasons } = await createRoutedHarness(defaultRoute, {
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("before_agent_start", () => ({
|
||||
message: { customType: "context", content: "Extra context.", display: false },
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harness.setResponses([fauxAssistantMessage("first"), fauxAssistantMessage("second")]);
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
await harness.session.prompt("again");
|
||||
|
||||
// The context ends with the extension message, but the request answers the user's prompt.
|
||||
expect(harness.session.messages.at(-2)?.role).toBe("custom");
|
||||
expect(reasons()).toEqual(["user", "user"]);
|
||||
});
|
||||
|
||||
it("ends the run with an error response when routing fails and keeps the last physical limits", async () => {
|
||||
let fail = false;
|
||||
const { harness } = await createRoutedHarness((request, ctx) => {
|
||||
if (fail) throw new Error("router unavailable");
|
||||
return defaultRoute(request, ctx);
|
||||
});
|
||||
harness.setResponses([fauxAssistantMessage("answer")]);
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
fail = true;
|
||||
await harness.session.prompt("again");
|
||||
|
||||
expect(harness.session.messages.at(-1)).toMatchObject({
|
||||
role: "assistant",
|
||||
provider: "router",
|
||||
model: "auto",
|
||||
stopReason: "error",
|
||||
errorMessage: expect.stringContaining("router unavailable"),
|
||||
});
|
||||
expect(harness.faux.state.callCount).toBe(1);
|
||||
// The failed attempt names the virtual model, whose declared window is 1k; the large model's 50k applies.
|
||||
expect(harness.session.getContextUsage()?.contextWindow).toBe(50_000);
|
||||
});
|
||||
|
||||
it("checks compaction against the physical model that produced the response", async () => {
|
||||
const { harness } = await createRoutedHarness();
|
||||
harness.setResponses([fauxAssistantMessage("short answer"), fauxAssistantMessage("long answer")]);
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
// About 20k tokens exceed the virtual model's declared 1k window but fit the large model's 50k.
|
||||
await harness.session.prompt("x".repeat(80_000));
|
||||
|
||||
expect(harness.eventsOfType("compaction_start")).toHaveLength(0);
|
||||
});
|
||||
|
||||
it("compacts before a request routed to a model with a smaller window", async () => {
|
||||
const { harness, dispatched } = await createRoutedHarness(defaultRoute, {
|
||||
settings: { compaction: { keepRecentTokens: 1, reserveTokens: 0 } },
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", async ({ preparation: { firstKeptEntryId, tokensBefore } }) => ({
|
||||
compaction: { summary: "compacted", firstKeptEntryId, tokensBefore },
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
let compactedBeforeSmall = false;
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("y".repeat(8000)),
|
||||
() => {
|
||||
compactedBeforeSmall = harness.eventsOfType("compaction_end").length === 1;
|
||||
return fauxAssistantMessage("small answer");
|
||||
},
|
||||
]);
|
||||
await harness.session.prompt("hello");
|
||||
expect(harness.eventsOfType("compaction_start")).toHaveLength(0);
|
||||
|
||||
// About 2k tokens fit the large model that answered last, but not the small model's 1k window.
|
||||
harness.session.setThinkingLevel("low");
|
||||
await harness.session.prompt("next");
|
||||
|
||||
expect(harness.eventsOfType("compaction_start").map((event) => event.reason)).toEqual(["threshold"]);
|
||||
expect(compactedBeforeSmall).toBe(true);
|
||||
expect(dispatched().at(-1)).toBe("faux/small:off");
|
||||
});
|
||||
|
||||
it("compacts between turns of a run when the next request is routed to a smaller window", async () => {
|
||||
const route: Route = (request, ctx) =>
|
||||
request.reason === "continuation"
|
||||
? { model: ctx.modelRegistry.find("faux", "small")!, thinkingLevel: "off" }
|
||||
: defaultRoute(request, ctx);
|
||||
const { harness, dispatched } = await createRoutedHarness(route, {
|
||||
settings: { compaction: { keepRecentTokens: 1, reserveTokens: 0 } },
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", async ({ preparation: { firstKeptEntryId, tokensBefore } }) => ({
|
||||
compaction: { summary: "compacted", firstKeptEntryId, tokensBefore },
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
let compactedBeforeSmall = false;
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(fauxToolCall("echo", { text: "hi" }), { stopReason: "toolUse" }),
|
||||
() => {
|
||||
compactedBeforeSmall = harness.eventsOfType("compaction_end").length === 1;
|
||||
return fauxAssistantMessage("small answer");
|
||||
},
|
||||
]);
|
||||
|
||||
// About 2k tokens fit the large model of the first turn, but not the small model of the second.
|
||||
await harness.session.prompt("x".repeat(8000));
|
||||
|
||||
expect(harness.eventsOfType("compaction_start").map((event) => event.reason)).toEqual(["threshold"]);
|
||||
expect(compactedBeforeSmall).toBe(true);
|
||||
expect(dispatched()).toEqual(["faux/large:high", "faux/small:off"]);
|
||||
});
|
||||
|
||||
it("projects the session once per request under a virtual selection", async () => {
|
||||
const { harness } = await createRoutedHarness();
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(fauxToolCall("echo", { text: "hi" }), { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("done"),
|
||||
]);
|
||||
const projections = vi.spyOn(harness.sessionManager, "buildSessionProjection");
|
||||
|
||||
await harness.session.prompt("hello");
|
||||
|
||||
// One per request, one between the turns, and one for the compaction check after the run.
|
||||
expect(projections).toHaveBeenCalledTimes(4);
|
||||
});
|
||||
|
||||
it("stores router state on the branch and passes it to later requests", async () => {
|
||||
const states: unknown[] = [];
|
||||
const { harness, reasons } = await createRoutedHarness(
|
||||
(request, ctx) => {
|
||||
states.push(request.state);
|
||||
const turns = (request.state as { turns: number } | undefined)?.turns ?? 0;
|
||||
const route = defaultRoute(request, ctx);
|
||||
// Returning request.state keeps it without storing it again. Direct requests neither get nor store state.
|
||||
if (request.reason === "continuation") return { ...route, state: request.state };
|
||||
return { ...route, state: { turns: turns + 1 } };
|
||||
},
|
||||
{ settings: { compaction: { keepRecentTokens: 1 } } },
|
||||
);
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage(fauxToolCall("echo", { text: "hi" }), { stopReason: "toolUse" }),
|
||||
fauxAssistantMessage("first"),
|
||||
fauxAssistantMessage("second"),
|
||||
fauxAssistantMessage("summary"),
|
||||
fauxAssistantMessage("summary"),
|
||||
]);
|
||||
|
||||
await harness.session.prompt("one");
|
||||
await harness.session.prompt("two");
|
||||
const stored = () =>
|
||||
harness.sessionManager
|
||||
.getBranch()
|
||||
.flatMap((entry) =>
|
||||
entry.type === "custom" && entry.customType === VIRTUAL_MODEL_STATE_ENTRY ? [entry.data] : [],
|
||||
);
|
||||
expect(stored()).toEqual([
|
||||
{ provider: "router", modelId: "auto", state: { turns: 1 } },
|
||||
{ provider: "router", modelId: "auto", state: { turns: 2 } },
|
||||
]);
|
||||
|
||||
await harness.session.compact();
|
||||
|
||||
expect(reasons()).toEqual(["user", "continuation", "user", "direct"]);
|
||||
expect(states).toEqual([undefined, { turns: 1 }, { turns: 1 }, undefined]);
|
||||
expect(stored()).toHaveLength(2);
|
||||
});
|
||||
|
||||
it("does not route compactions that an extension supplies", async () => {
|
||||
const route: Route = (request, ctx) => {
|
||||
if (request.reason === "direct") throw new Error("router unavailable");
|
||||
return defaultRoute(request, ctx);
|
||||
};
|
||||
const { harness, reasons } = await createRoutedHarness(route, {
|
||||
settings: { compaction: { keepRecentTokens: 1 } },
|
||||
extensionFactories: [
|
||||
(pi) => {
|
||||
pi.on("session_before_compact", async ({ preparation: { firstKeptEntryId, tokensBefore } }) => ({
|
||||
compaction: { summary: "extension summary", firstKeptEntryId, tokensBefore },
|
||||
}));
|
||||
},
|
||||
],
|
||||
});
|
||||
harness.setResponses([fauxAssistantMessage("first answer"), fauxAssistantMessage("second answer")]);
|
||||
await harness.session.prompt("first");
|
||||
await harness.session.prompt("second");
|
||||
|
||||
const result = await harness.session.compact();
|
||||
|
||||
expect(result.summary).toBe("extension summary");
|
||||
expect(reasons()).toEqual(["user", "user"]);
|
||||
});
|
||||
|
||||
it("routes compaction summaries before sizing them", async () => {
|
||||
const summaries: string[] = [];
|
||||
const { harness, reasons } = await createRoutedHarness(defaultRoute, {
|
||||
settings: { compaction: { keepRecentTokens: 1 } },
|
||||
});
|
||||
const summary: FauxResponseStep = (_context, options, _state, model) => {
|
||||
summaries.push(`${model.id}:${options?.reasoning ?? "off"}:${options?.maxTokens}`);
|
||||
return fauxAssistantMessage("summary");
|
||||
};
|
||||
// Compaction summarizes the history and the split turn prefix with one routed model.
|
||||
harness.setResponses([
|
||||
fauxAssistantMessage("first answer"),
|
||||
fauxAssistantMessage("second answer"),
|
||||
summary,
|
||||
summary,
|
||||
]);
|
||||
await harness.session.prompt("first");
|
||||
await harness.session.prompt("second");
|
||||
|
||||
const result = await harness.session.compact();
|
||||
|
||||
expect(result.summary).toContain("summary");
|
||||
expect(reasons()).toEqual(["user", "user", "direct"]);
|
||||
// The router's thinking level applies, and the output budget respects the large model's 4000 tokens.
|
||||
expect(summaries).toEqual(["large:low:4000", "large:low:4000"]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user