import { type AssistantMessage, fauxAssistantMessage, fauxText, fauxToolCall, Type } from "@earendil-works/pi-ai"; import { AgentDoc, type Conversation, defineTool, MemoryStorage, type NestedToolExecutionResult, type ToolRegistration, } from "@earendil-works/pi-durable"; import { describe, expect, it } from "vitest"; import { type ChatSetup, chatSetup, openChat } from "./chat-support.ts"; import { addTool } from "./harness-support.ts"; import { context } from "./session-support.ts"; function tool(name: string, extra: Partial = {}): ToolRegistration { return defineTool({ name, description: name, parameters: Type.Object({}), execute: async () => ({ output: [{ type: "text", text: `${name} ran` }] }), ...extra, }); } const DONE = fauxAssistantMessage([fauxText("done")]); function call(name: string, id = "c1"): AssistantMessage { return fauxAssistantMessage([fauxToolCall(name, {}, { id })], { stopReason: "toolUse" }); } /** A tool that calls each of `names` as a nested call and records what it got back. */ function prober(names: readonly string[]) { const received = new Map(); const registration = defineTool({ name: "probe", description: "probe", parameters: Type.Object({}), execute: async (_args, api, callContext) => { for (const name of names) received.set(name, await api.executeTool(name, {}, callContext)); return {}; }, }); return { registration, received }; } async function offered(root: Conversation): Promise { return (await root.agent(context)).tools.map((each) => each.name); } async function callable(root: Conversation): Promise { return (await root.agent(context)).callable.map((each) => each.name); } async function setupWith(tools: readonly ToolRegistration[]): Promise { const setup = chatSetup(); for (const each of tools) addTool(setup.registry, each); return setup; } describe("who may call a tool", () => { it("offers the model only tools it may call, and lets tools call only tools they may call", async () => { const script = tool("script", { callers: ["model"] }); const hidden = tool("hidden", { callers: ["tools"] }); const both = tool("both"); const probe = prober(["script", "hidden", "both"]); const setup = await setupWith([script, hidden, both, probe.registration]); setup.faux.setResponses([call("probe"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); expect(await offered(root)).toEqual(["script", "both", "probe"]); expect(await callable(root)).toEqual(["hidden", "both", "probe"]); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(probe.received.get("script")?.diagnostics[0]?.code).toBe("tool_unavailable"); expect(probe.received.get("hidden")).toMatchObject({ isError: false, structuredOutput: "hidden ran" }); expect(probe.received.get("both")).toMatchObject({ isError: false, structuredOutput: "both ran" }); await harness.close(context); }); it("narrows what the model is offered with modelTools, leaving the rest callable by tools", async () => { const read = tool("read"); const bash = tool("bash"); const probe = prober(["read", "bash"]); const setup = await setupWith([read, bash, probe.registration]); setup.faux.setResponses([call("probe"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); await root.configure({ modelTools: [probe.registration] }, context); expect(await offered(root)).toEqual(["probe"]); expect(await callable(root)).toEqual(["read", "bash", "probe"]); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(probe.received.get("read")?.isError).toBe(false); expect(probe.received.get("bash")?.isError).toBe(false); await root.configure({ modelTools: { remove: [bash] } }, context); expect(await offered(root)).toEqual(["read", "probe"]); expect((await harness.snapshot(AgentDoc, root.id, context))?.modelTools).toEqual({ remove: ["bash"] }); await root.configure({ modelTools: null }, context); expect(await offered(root)).toEqual(["read", "bash", "probe"]); await harness.close(context); }); it("never offers a tool that is not enabled, nor lets tools call it", async () => { const read = tool("read"); const bash = tool("bash"); const probe = prober(["bash"]); const setup = await setupWith([read, bash, probe.registration]); setup.faux.setResponses([call("probe"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); // modelTools cannot bring back what tools disabled. await root.configure({ tools: { remove: [bash] }, modelTools: [bash, probe.registration] }, context); expect(await offered(root)).toEqual(["probe"]); expect(await callable(root)).toEqual(["read", "probe"]); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect(probe.received.get("bash")?.diagnostics[0]?.code).toBe("tool_unavailable"); await harness.close(context); }); it("answers a model-issued call of a tool it was not offered as unavailable", async () => { const read = tool("read"); const bash = tool("bash"); const setup = await setupWith([read, bash]); setup.faux.setResponses([call("bash"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); await root.configure({ modelTools: [read] }, context); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); const entries = await root.entries({}, 100, undefined, context); const result = entries.items.find((entry) => entry.kind === "pi.tool-result"); expect(JSON.stringify(result?.model)).toContain("Tool bash is not available"); await harness.close(context); }); it("applies addTools to modelTools too, so the model is offered the added tools", async () => { const extra = tool("extra"); const grow = tool("grow", { execute: async () => ({ output: [], control: { addTools: ["extra"] } }), }); const setup = await setupWith([extra, grow]); setup.faux.setResponses([call("grow"), DONE]); const { harness, root } = await openChat(new MemoryStorage(), setup); await root.configure({ modelTools: [grow] }, context); await (await root.submit({ type: "input", content: "go" }, context)).wait(context); expect((await harness.snapshot(AgentDoc, root.id, context))?.modelTools).toEqual(["grow", "extra"]); expect(await offered(root)).toEqual(["grow", "extra"]); await harness.close(context); }); });