alexion
6081e3e325
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
292 lines
9.4 KiB
TypeScript
292 lines
9.4 KiB
TypeScript
import { describe, expect, it, vi } from "vitest";
|
|
import { classify } from "../src/api/openai-decisions.ts";
|
|
import type { ClassifierContext, ClassifierModel, ImageContent } from "../src/types.ts";
|
|
|
|
const model: ClassifierModel<"openai-decisions"> = {
|
|
type: "classifier",
|
|
id: "gpt-6-luna",
|
|
name: "GPT-6 Luna",
|
|
api: "openai-decisions",
|
|
provider: "openai",
|
|
baseUrl: "https://api.openai.com/v1",
|
|
input: ["text", "image"],
|
|
cost: {
|
|
input: 0.1,
|
|
output: 0,
|
|
cacheRead: 0,
|
|
cacheWrite: 0,
|
|
tiers: [{ inputTokensAbove: 272000, input: 0.2, output: 0, cacheRead: 0, cacheWrite: 0 }],
|
|
},
|
|
contextWindow: 922000,
|
|
};
|
|
|
|
const context: ClassifierContext = {
|
|
state: { text: "The deployment succeeded, thank you." },
|
|
questions: {
|
|
category: {
|
|
type: "choice",
|
|
instructions: "Classify the message",
|
|
criteria: { success: "Successful", failure: "" },
|
|
},
|
|
satisfaction: {
|
|
type: "score",
|
|
instructions: "Score satisfaction",
|
|
criteria: ["low", "neutral", "high"],
|
|
},
|
|
approved: {
|
|
type: "bool",
|
|
instructions: "Does the user approve?",
|
|
criteria: { true: "Approval", false: "No approval" },
|
|
},
|
|
},
|
|
};
|
|
|
|
/** Response shape from the API reference and live `gpt-6-luna` requests. */
|
|
const wireAnswers = [
|
|
{
|
|
type: "choice",
|
|
name: "category",
|
|
choice: "success",
|
|
probabilities: [
|
|
{ value: "success", probability: 0.9 },
|
|
{ value: "failure", probability: 0.1 },
|
|
],
|
|
confidence: 0.8,
|
|
},
|
|
{
|
|
type: "score",
|
|
name: "satisfaction",
|
|
score: 1.8,
|
|
probabilities: [
|
|
{ value: 0, label: "low", probability: 0.05 },
|
|
{ value: 1, label: "neutral", probability: 0.1 },
|
|
{ value: 2, label: "high", probability: 0.85 },
|
|
],
|
|
confidence: 0.7,
|
|
},
|
|
{ type: "predicate", name: "approved", probability: 0.95 },
|
|
];
|
|
|
|
const wireUsage = {
|
|
input_tokens: 164,
|
|
input_tokens_details: { cached_tokens: 0, cache_write_tokens: 0 },
|
|
output_tokens: 0,
|
|
output_tokens_details: { reasoning_tokens: 0 },
|
|
total_tokens: 164,
|
|
};
|
|
|
|
const image: ImageContent = { type: "image", data: "aW1hZ2U=", mimeType: "image/png" };
|
|
|
|
describe("OpenAI Decisions", () => {
|
|
it("maps questions to Decisions types and answers back by name", async () => {
|
|
const fetch = vi.fn(async (_input: string | URL | Request, _init?: RequestInit) =>
|
|
// Answers out of question order: they are matched by name.
|
|
Response.json({ model: "gpt-6-luna", answers: [...wireAnswers].reverse(), usage: wireUsage }),
|
|
);
|
|
|
|
const result = await classify(model, context, { apiKey: "secret", fetch, temperature: 1.5 });
|
|
|
|
expect(fetch).toHaveBeenCalledOnce();
|
|
const [url, init] = fetch.mock.calls[0]!;
|
|
expect(String(url)).toBe("https://api.openai.com/v1/decisions");
|
|
expect(new Headers(init?.headers).get("authorization")).toBe("Bearer secret");
|
|
expect(JSON.parse(String(init?.body))).toEqual({
|
|
model: "gpt-6-luna",
|
|
input: JSON.stringify(context.state),
|
|
questions: [
|
|
{
|
|
type: "choice",
|
|
name: "category",
|
|
instructions: "Classify the message",
|
|
// Empty descriptions are omitted.
|
|
choices: [{ value: "success", description: "Successful" }, { value: "failure" }],
|
|
},
|
|
{
|
|
type: "score",
|
|
name: "satisfaction",
|
|
instructions: "Score satisfaction",
|
|
levels: [{ label: "low" }, { label: "neutral" }, { label: "high" }],
|
|
},
|
|
{
|
|
type: "predicate",
|
|
name: "approved",
|
|
instructions: "Does the user approve?\n\nTrue means: Approval\nFalse means: No approval",
|
|
},
|
|
],
|
|
});
|
|
expect(result.stopReason).toBe("stop");
|
|
expect(result.answers).toEqual({
|
|
category: {
|
|
type: "choice",
|
|
choice: "success",
|
|
probabilities: { success: 0.9, failure: 0.1 },
|
|
confidence: 0.8,
|
|
},
|
|
satisfaction: { type: "score", score: 1.8, confidence: 0.7 },
|
|
approved: { type: "bool", probability: 0.95 },
|
|
});
|
|
expect(result.usage).toMatchObject({ input: 164, output: 0, cacheRead: 0, totalTokens: 164 });
|
|
expect(result.usage?.cost.total).toBeCloseTo(0.0000164, 12);
|
|
});
|
|
|
|
it("prices long-context requests at the long-context input rate", async () => {
|
|
const result = await classify(model, context, {
|
|
apiKey: "secret",
|
|
fetch: async () => Response.json({ answers: wireAnswers, usage: { input_tokens: 300000, output_tokens: 0 } }),
|
|
});
|
|
|
|
expect(result.usage?.cost.total).toBeCloseTo(0.06, 12);
|
|
});
|
|
|
|
it("sends images after the state in one user message", async () => {
|
|
const fetch = vi.fn(async (_input: string | URL | Request, _init?: RequestInit) =>
|
|
Response.json({ answers: wireAnswers }),
|
|
);
|
|
|
|
const result = await classify(
|
|
model,
|
|
{ ...context, images: [image, { ...image, mimeType: "image/jpeg" }] },
|
|
{
|
|
apiKey: "secret",
|
|
fetch,
|
|
},
|
|
);
|
|
|
|
expect(result.stopReason).toBe("stop");
|
|
expect(JSON.parse(String(fetch.mock.calls[0]?.[1]?.body)).input).toEqual([
|
|
{
|
|
role: "user",
|
|
content: [
|
|
{ type: "input_text", text: JSON.stringify(context.state) },
|
|
{ type: "input_image", image_url: "data:image/png;base64,aW1hZ2U=" },
|
|
{ type: "input_image", image_url: "data:image/jpeg;base64,aW1hZ2U=" },
|
|
],
|
|
},
|
|
]);
|
|
});
|
|
|
|
it("rejects more than 128 images before sending", async () => {
|
|
const fetch = vi.fn(async () => Response.json({ answers: wireAnswers }));
|
|
|
|
const result = await classify(
|
|
model,
|
|
{ ...context, images: Array.from({ length: 129 }, () => image) },
|
|
{
|
|
apiKey: "secret",
|
|
fetch,
|
|
},
|
|
);
|
|
|
|
expect(fetch).not.toHaveBeenCalled();
|
|
expect(result.stopReason).toBe("error");
|
|
expect(result.errorMessage).toContain("at most 128 images, got 129");
|
|
});
|
|
|
|
it("fails the result when a question is refused and keeps the billed usage", async () => {
|
|
const result = await classify(model, context, {
|
|
apiKey: "secret",
|
|
fetch: async () =>
|
|
Response.json({
|
|
answers: [wireAnswers[0], wireAnswers[1], { type: "refusal", name: "approved" }],
|
|
usage: wireUsage,
|
|
}),
|
|
});
|
|
|
|
expect(result.stopReason).toBe("error");
|
|
expect(result.answers).toEqual({});
|
|
expect(result.errorMessage).toBe("OpenAI Decisions refused to answer approved");
|
|
expect(result.usage).toMatchObject({ input: 164 });
|
|
});
|
|
|
|
it("returns missing and mistyped answers as classifier errors", async () => {
|
|
const missing = await classify(model, context, {
|
|
apiKey: "secret",
|
|
fetch: async () => Response.json({ answers: wireAnswers.slice(0, 2) }),
|
|
});
|
|
const mistyped = await classify(model, context, {
|
|
apiKey: "secret",
|
|
fetch: async () =>
|
|
Response.json({ answers: [wireAnswers[0], wireAnswers[1], { type: "score", name: "approved" }] }),
|
|
});
|
|
|
|
expect(missing.stopReason).toBe("error");
|
|
expect(missing.errorMessage).toContain("did not return an answer for approved");
|
|
expect(mistyped.stopReason).toBe("error");
|
|
expect(mistyped.errorMessage).toContain("did not return a predicate answer for approved");
|
|
});
|
|
|
|
it("preserves prototype-sensitive question IDs in answers", async () => {
|
|
const prototypeContext: ClassifierContext = {
|
|
state: {},
|
|
questions: JSON.parse(
|
|
'{"__proto__":{"type":"bool","instructions":"Is this true?","criteria":{"true":"Yes","false":"No"}}}',
|
|
),
|
|
};
|
|
const result = await classify(model, prototypeContext, {
|
|
apiKey: "secret",
|
|
fetch: async () => Response.json({ answers: [{ type: "predicate", name: "__proto__", probability: 0.75 }] }),
|
|
});
|
|
|
|
expect(result.stopReason).toBe("stop");
|
|
expect(Object.hasOwn(result.answers, "__proto__")).toBe(true);
|
|
expect(result.answers.__proto__).toEqual({ type: "bool", probability: 0.75 });
|
|
});
|
|
|
|
it("does not retry gateway timeouts and explains them instead of returning the HTML page", async () => {
|
|
const fetch = vi.fn(
|
|
async () =>
|
|
new Response("<!DOCTYPE html><html>Gateway time-out</html>", {
|
|
status: 504,
|
|
headers: { "retry-after-ms": "0" },
|
|
}),
|
|
);
|
|
// Default retries: the same input would time out again.
|
|
const result = await classify(model, context, { apiKey: "secret", fetch });
|
|
|
|
expect(fetch).toHaveBeenCalledOnce();
|
|
expect(result.stopReason).toBe("error");
|
|
expect(result.errorMessage).toContain("OpenAI Decisions error (504): the request timed out at the gateway");
|
|
expect(result.errorMessage).not.toContain("<html>");
|
|
});
|
|
|
|
it("still retries other server errors", async () => {
|
|
let attempt = 0;
|
|
const result = await classify(model, context, {
|
|
apiKey: "secret",
|
|
fetch: async () =>
|
|
++attempt === 1
|
|
? new Response("busy", { status: 503, headers: { "retry-after-ms": "0" } })
|
|
: Response.json({ answers: wireAnswers }),
|
|
});
|
|
|
|
expect(attempt).toBe(2);
|
|
expect(result.stopReason).toBe("stop");
|
|
});
|
|
|
|
it("includes the API error body for other HTTP failures", async () => {
|
|
const result = await classify(model, context, {
|
|
apiKey: "secret",
|
|
maxRetries: 0,
|
|
fetch: async () =>
|
|
Response.json(
|
|
{ error: { message: "Decision input exceeds the token limit.", type: "invalid_request_error" } },
|
|
{ status: 400 },
|
|
),
|
|
});
|
|
|
|
expect(result.stopReason).toBe("error");
|
|
expect(result.errorMessage).toContain("OpenAI Decisions error (400)");
|
|
expect(result.errorMessage).toContain("Decision input exceeds the token limit.");
|
|
});
|
|
|
|
it("rejects models for other classifier APIs and missing API keys", async () => {
|
|
const fetch = vi.fn(async () => Response.json({ answers: wireAnswers }));
|
|
const otherApi = await classify({ ...model, api: "typesafe-system-one" }, context, { apiKey: "secret", fetch });
|
|
const noKey = await classify(model, context, { fetch });
|
|
|
|
expect(fetch).not.toHaveBeenCalled();
|
|
expect(otherApi.errorMessage).toContain("Unsupported classifier API: typesafe-system-one");
|
|
expect(noKey.errorMessage).toContain("No API key for provider: openai");
|
|
});
|
|
});
|