alexion/dev
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

chore(vendor): import pi 1.3.0

Browse Source 
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree.

git-subtree-dir: third_party/pi

git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
alexion
2026-10-15 17:58:48 -04:00
parent 719370c8f7
commit 6081e3e325
1743 changed files with 205460 additions and 131705 deletions
Download Patch File Download Diff File Expand all files Collapse all files

98
third_party/pi/packages/ai/scripts/model-data.ts vendored
View File

@@ -2,7 +2,7 @@ import { createHash } from "node:crypto";
import { existsSync, readFileSync, readdirSync, statSync } from "node:fs";
import { join } from "node:path";
export const MODEL_DATA_SCHEMA_VERSION = 3;
export const MODEL_DATA_SCHEMA_VERSION = 6;
export const MODEL_DATA_MANIFEST_FILE = ".manifest.json";
export type ModelDataStructure = Record<string, Record<string, string>>;
@@ -15,7 +15,7 @@ export interface ModelDataManifest {
}
const MODEL_DATA_IMPORT_PATTERN =
/^import \{ [A-Z][A-Z0-9_]*_MODELS \} from "\.\/providers\/([^"/]+)\.models\.ts";$/gm;
/^import \{ [A-Z][A-Z0-9_]*_CLASSIFIER_MODELS, [A-Z][A-Z0-9_]*_IMAGE_MODELS, [A-Z][A-Z0-9_]*_MODELS \} from "\.\/providers\/([^"/]+)\.models\.ts";$/gm;
function sha256(value: string): string {
return createHash("sha256").update(value).digest("hex");
@@ -50,6 +50,10 @@ function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function isModalityList(value: unknown): value is ("text" | "image")[] {
return Array.isArray(value) && value.length > 0 && value.every((entry) => entry === "text" || entry === "image");
}
function readJsonObject(path: string, description: string, errors: string[]): Record<string, unknown> | undefined {
let parsed: unknown;
try {
@@ -73,9 +77,9 @@ function readProviderStructure(path: string, providerId: string): Record<string,
const models = new Map<string, string>();
for (const [api, value] of Object.entries(groups)) {
if (!isRecord(value)) throw new Error(`${path} API group ${JSON.stringify(api)} must be an object`);
for (const modelId of Object.keys(value)) {
if (models.has(modelId)) throw new Error(`${path} contains model ${modelId} in more than one API group`);
models.set(modelId, api);
for (const modelKey of Object.keys(value)) {
if (models.has(modelKey)) throw new Error(`${path} contains ${modelKey} in more than one API group`);
models.set(modelKey, api);
}
}
if (models.size === 0) throw new Error(`${path} contains no generated model data`);
@@ -158,19 +162,36 @@ function validateModelValue(
}
if (typeof value.name !== "string" || value.name.length === 0) errors.push(`${label} has no model name`);
if (typeof value.baseUrl !== "string") errors.push(`${label} has no baseUrl string`);
if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`);
if (
!Array.isArray(value.input) ||
value.input.length === 0 ||
value.input.some((entry) => entry !== "text" && entry !== "image")
) {
errors.push(`${label} has invalid input modalities`);
if (!isModalityList(value.input)) errors.push(`${label} has invalid input modalities`);
if (value.type === "image") {
if (!isModalityList(value.output) || !value.output.includes("image")) {
errors.push(`${label} has invalid output modalities`);
}
} else if (value.output !== undefined) {
errors.push(`${label} has unsupported output modalities`);
}
if (typeof value.contextWindow !== "number" || !Number.isFinite(value.contextWindow) || value.contextWindow <= 0) {
errors.push(`${label} has invalid contextWindow`);
}
if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) {
errors.push(`${label} has invalid maxTokens`);
if (value.type === "chat") {
if (typeof value.reasoning !== "boolean") errors.push(`${label} has no reasoning boolean`);
if (
typeof value.contextWindow !== "number" ||
!Number.isFinite(value.contextWindow) ||
value.contextWindow <= 0
) {
errors.push(`${label} has invalid contextWindow`);
}
if (typeof value.maxTokens !== "number" || !Number.isFinite(value.maxTokens) || value.maxTokens <= 0) {
errors.push(`${label} has invalid maxTokens`);
}
} else if (value.type === "classifier") {
if (
typeof value.contextWindow !== "number" ||
!Number.isFinite(value.contextWindow) ||
value.contextWindow <= 0
) {
errors.push(`${label} has invalid contextWindow`);
}
} else if (value.type !== "image") {
errors.push(`${label} has type ${JSON.stringify(value.type)}, expected "chat", "image", or "classifier"`);
}
if (!isRecord(value.cost)) {
errors.push(`${label} has invalid cost metadata`);
@@ -246,13 +267,18 @@ export function validateModelDataDirectory(structure: ModelDataStructure, dataDi
errors.push(`${filename} API group ${JSON.stringify(api)} must be an object`);
continue;
}
for (const [modelId, model] of Object.entries(value)) {
if (actualModels.has(modelId)) {
errors.push(`${providerId}/${modelId} appears in more than one API group`);
for (const [modelKey, model] of Object.entries(value)) {
if (actualModels.has(modelKey)) {
errors.push(`${providerId}/${modelKey} appears in more than one API group`);
continue;
}
actualModels.set(modelId, api);
actualModels.set(modelKey, api);
const separator = modelKey.indexOf(":");
const modelId = separator >= 0 ? modelKey.slice(separator + 1) : modelKey;
validateModelValue(model, providerId, modelId, api, errors);
if (isRecord(model) && modelKey !== `${String(model.type)}:${String(model.id)}`) {
errors.push(`${providerId}/${modelKey} has mismatched type/id identity`);
}
}
}
@@ -278,3 +304,33 @@ export function validateGeneratedModelData(packageRoot: string): void {
const structure = readModelDataStructure(packageRoot);
validateModelDataDirectory(structure, join(packageRoot, "src", "providers", "data"));
}
export interface ModelCatalogEntry {
type: string;
id: string;
api: string;
}
/**
* Group one provider's typed catalog entries by API and key them by `type:id`,
* the layout of `src/providers/data/<provider>.json`.
*/
export function groupProviderModelData<T extends ModelCatalogEntry>(
providerId: string,
models: readonly T[],
): { groups: Record<string, Record<string, T>>; structure: Record<string, string> } {
const groups: Record<string, Record<string, T>> = {};
const structure: Record<string, string> = {};
for (const api of Array.from(new Set(models.map((model) => model.api))).sort()) {
const group: Record<string, T> = {};
for (const model of models) {
if (model.api !== api) continue;
const identity = `${model.type}:${model.id}`;
if (group[identity]) throw new Error(`${providerId}/${identity} has duplicate ${api} catalog entries`);
group[identity] = model;
structure[identity] = api;
}
groups[api] = group;
}
return { groups, structure };
}