alexion/dev
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

chore(vendor): import pi 1.3.0

Browse Source 
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
alexion
2026-10-15 17:58:48 -04:00
parent 719370c8f7
commit 6081e3e325
1743 changed files with 205460 additions and 131705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

190
third_party/pi/packages/ai/test/model-data-validation.test.ts vendored
View File

@@ -2,6 +2,7 @@ import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "nod
import { tmpdir } from "node:os";
import { join } from "node:path";
import { afterEach, describe, expect, it } from "vitest";
import { hydrateModelCatalog } from "../scripts/hydrate-model-catalog.ts";
import {
assertExactModelIds,
createModelDataManifest,
@@ -32,20 +33,21 @@ function createFixture(): {
mkdirSync(dataDir, { recursive: true });
writeFileSync(
join(packageRoot, "src", "models.generated.ts"),
'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\n',
'import { TEST_PROVIDER_CLASSIFIER_MODELS, TEST_PROVIDER_IMAGE_MODELS, TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\n',
);
writeFileSync(
join(providersDir, "test-provider.models.ts"),
'import values from "./data/test-provider.json" with { type: "json" };\nimport { flattenModelCatalog, type ModelCatalog } from "../model-catalog.ts";\n\nexport const TEST_PROVIDER_MODELS: ModelCatalog<typeof values, "test-provider"> =\n\tflattenModelCatalog("test-provider", values);\n',
'import values from "./data/test-provider.json" with { type: "json" };\n',
);
const structure: ModelDataStructure = {
"test-provider": {
"model-a": "openai-completions",
"chat:model-a": "openai-completions",
},
};
const values: Record<string, unknown> = {
"model-a": {
"chat:model-a": {
type: "chat",
id: "model-a",
name: "Model A",
api: "openai-completions",
@@ -77,6 +79,104 @@ function writeFixtureData(
writeFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), `${JSON.stringify(manifest)}\n`);
}
describe("published model catalog hydration", () => {
function chatModel(values: Record<string, unknown>): Record<string, unknown> {
return values["chat:model-a"] as Record<string, unknown>;
}
it("hydrates a clean checkout deterministically without changing generated TypeScript", () => {
const { dataDir, packageRoot, structure, values } = createFixture();
const aggregatorPath = join(packageRoot, "src", "models.generated.ts");
const aggregator = readFileSync(aggregatorPath, "utf8");
const catalogPath = join(packageRoot, "catalog.json");
const models = [chatModel(values)];
writeFileSync(catalogPath, JSON.stringify({ "test-provider": models, "extra-provider": models }));
rmSync(dataDir, { recursive: true });
hydrateModelCatalog(packageRoot, catalogPath);
expect(() => validateModelDataDirectory(structure, dataDir)).not.toThrow();
expect(readModelDataStructure(packageRoot)).toEqual(structure);
expect(readFileSync(aggregatorPath, "utf8")).toBe(aggregator);
const manifest = readFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), "utf8");
const provider = readFileSync(join(dataDir, "test-provider.json"), "utf8");
hydrateModelCatalog(packageRoot, catalogPath);
expect(readFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), "utf8")).toBe(manifest);
expect(readFileSync(join(dataDir, "test-provider.json"), "utf8")).toBe(provider);
});
it("groups models by API and keys them by type and id", () => {
const { packageRoot, values } = createFixture();
const catalogPath = join(packageRoot, "catalog.json");
const model = chatModel(values);
writeFileSync(
catalogPath,
JSON.stringify({
"test-provider": [model, { ...model, id: "model-b", api: "anthropic-messages" }],
}),
);
hydrateModelCatalog(packageRoot, catalogPath);
expect(readModelDataStructure(packageRoot)).toEqual({
"test-provider": { "chat:model-a": "openai-completions", "chat:model-b": "anthropic-messages" },
});
});
it("keys chat and image models with the same id separately", () => {
const { packageRoot, values } = createFixture();
const image = {
type: "image",
id: "model-a",
name: "Model A Image",
api: "openrouter-images",
provider: "test-provider",
baseUrl: "https://example.test/v1",
input: ["text"],
output: ["image"],
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
};
const catalogPath = join(packageRoot, "catalog.json");
writeFileSync(catalogPath, JSON.stringify({ "test-provider": [chatModel(values), image] }));
hydrateModelCatalog(packageRoot, catalogPath);
expect(readModelDataStructure(packageRoot)).toEqual({
"test-provider": { "chat:model-a": "openai-completions", "image:model-a": "openrouter-images" },
});
});
it("validates without replacing existing data", () => {
const { dataDir, packageRoot, values } = createFixture();
const original = readFileSync(join(dataDir, "test-provider.json"), "utf8");
const catalogPath = join(packageRoot, "catalog.json");
const model = chatModel(values);
writeFileSync(catalogPath, JSON.stringify({ "test-provider": [model, { ...model, id: "model-b" }] }));
hydrateModelCatalog(packageRoot, catalogPath, { validateOnly: true });
expect(readFileSync(join(dataDir, "test-provider.json"), "utf8")).toBe(original);
writeFileSync(catalogPath, JSON.stringify({ "other-provider": [model] }));
expect(() => hydrateModelCatalog(packageRoot, catalogPath, { validateOnly: true })).toThrow(/missing provider/);
});
it.each([null, [], {}, { "test-provider": [] }, { "test-provider": {} }, { "test-provider": [{ id: "x" }] }])(
"rejects incomplete catalogs without replacing existing data: %j",
(catalog) => {
const { dataDir, packageRoot } = createFixture();
const catalogPath = join(packageRoot, "catalog.json");
const original = readFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), "utf8");
writeFileSync(catalogPath, JSON.stringify(catalog));
expect(() => hydrateModelCatalog(packageRoot, catalogPath)).toThrow();
expect(readFileSync(join(dataDir, MODEL_DATA_MANIFEST_FILE), "utf8")).toBe(original);
},
);
it.each(["api", "provider", "cost"])("validates model %s before replacing existing data", (field) => {
const { dataDir, packageRoot, values } = createFixture();
const original = readFileSync(join(dataDir, "test-provider.json"), "utf8");
const model = chatModel(values);
model[field] = null;
const catalogPath = join(packageRoot, "catalog.json");
writeFileSync(catalogPath, JSON.stringify({ "test-provider": [model] }));
expect(() => hydrateModelCatalog(packageRoot, catalogPath)).toThrow();
expect(readFileSync(join(dataDir, "test-provider.json"), "utf8")).toBe(original);
});
});
describe("generated model data validation", () => {
it("rejects a missing upstream model from an exact generated allowlist", () => {
expect(() => assertExactModelIds("qwen-token-plan-individual", ["model-a", "model-b"], ["model-a"])).toThrow(
@@ -108,12 +208,90 @@ describe("generated model data validation", () => {
["api", "anthropic-messages", "has api"],
] as const)("rejects a wrong model %s", (field, value, expectedMessage) => {
const fixture = createFixture();
const model = fixture.values["model-a"] as Record<string, unknown>;
const model = fixture.values["chat:model-a"] as Record<string, unknown>;
model[field] = value;
writeFixtureData(fixture.dataDir, fixture.structure, fixture.values);
expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(expectedMessage);
});
it("rejects a model without a known type", () => {
const fixture = createFixture();
const model = fixture.values["chat:model-a"] as Record<string, unknown>;
delete model.type;
writeFixtureData(fixture.dataDir, fixture.structure, fixture.values);
expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(
'expected "chat", "image", or "classifier"',
);
});
it("validates image models with output modalities and without chat limits", () => {
const fixture = createFixture();
const structure: ModelDataStructure = { "test-provider": { "image:image-a": "test-images" } };
const image: Record<string, unknown> = {
type: "image",
id: "image-a",
name: "Image A",
api: "test-images",
provider: "test-provider",
baseUrl: "https://example.test/v1",
input: ["text"],
output: ["image", "text"],
cost: { input: 1, output: 2, cacheRead: 0, cacheWrite: 0 },
};
const validate = () => {
writeFixtureData(
fixture.dataDir,
structure,
{ "image:image-a": image },
MODEL_DATA_SCHEMA_VERSION,
"test-images",
);
validateModelDataDirectory(structure, fixture.dataDir);
};
expect(validate).not.toThrow();
delete image.output;
expect(validate).toThrow("invalid output modalities");
image.output = ["text"];
expect(validate).toThrow("invalid output modalities");
});
it("rejects output modalities on chat models", () => {
const fixture = createFixture();
const model = fixture.values["chat:model-a"] as Record<string, unknown>;
model.output = ["text"];
writeFixtureData(fixture.dataDir, fixture.structure, fixture.values);
expect(() => validateModelDataDirectory(fixture.structure, fixture.dataDir)).toThrow(
"unsupported output modalities",
);
});
it("validates classifier models without chat output limits", () => {
const fixture = createFixture();
const structure: ModelDataStructure = {
"test-provider": { "classifier:classifier-a": "test-classifier" },
};
const classifier = {
type: "classifier",
id: "classifier-a",
name: "Classifier A",
api: "test-classifier",
provider: "test-provider",
baseUrl: "https://example.test/v1",
input: ["text"],
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
contextWindow: 1000,
};
writeFixtureData(
fixture.dataDir,
structure,
{ "classifier:classifier-a": classifier },
MODEL_DATA_SCHEMA_VERSION,
"test-classifier",
);
expect(() => validateModelDataDirectory(structure, fixture.dataDir)).not.toThrow();
});
it("rejects a model in the wrong API group", () => {
const fixture = createFixture();
writeFixtureData(
@@ -170,7 +348,7 @@ describe("generated model data validation", () => {
const { packageRoot } = createFixture();
writeFileSync(
join(packageRoot, "src", "models.generated.ts"),
'import { TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\nimport { MISSING_MODELS } from "./providers/missing.models.ts";\n',
'import { TEST_PROVIDER_CLASSIFIER_MODELS, TEST_PROVIDER_IMAGE_MODELS, TEST_PROVIDER_MODELS } from "./providers/test-provider.models.ts";\nimport { MISSING_CLASSIFIER_MODELS, MISSING_IMAGE_MODELS, MISSING_MODELS } from "./providers/missing.models.ts";\n',
);
expect(() => readModelDataStructure(packageRoot)).toThrow("aggregator and provider shards do not match");
});