chore(vendor): import pi 1.3.0
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
189
third_party/pi/scripts/publish-model-catalog.mjs
vendored
189
third_party/pi/scripts/publish-model-catalog.mjs
vendored
@@ -12,17 +12,29 @@ import { tmpdir } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { spawnSync } from "node:child_process";
|
||||
import { isDeepStrictEqual } from "node:util";
|
||||
import {
|
||||
compareModelCatalogPiVersions,
|
||||
getModelCatalogArtifactKey,
|
||||
getModelCatalogProviderKey,
|
||||
MODEL_CATALOG_INDEX_KEY,
|
||||
MODEL_CATALOG_PREFIX,
|
||||
MODEL_CATALOG_SCHEMA_VERSION,
|
||||
parseModelCatalogIndex,
|
||||
} from "./model-catalog-protocol.ts";
|
||||
|
||||
const CATALOG_SCHEMA_VERSION = 1;
|
||||
const CATALOG_PREFIX = `models/v${CATALOG_SCHEMA_VERSION}`;
|
||||
const CATALOG_INDEX_KEY = `${CATALOG_PREFIX}/index.json`;
|
||||
// The storage layout, index format, and version ordering are defined in
|
||||
// model-catalog-protocol.ts, which pi.dev shares to serve these artifacts.
|
||||
// Bump this only when generated model metadata requires behavior unavailable in older pi clients.
|
||||
const MINIMUM_PI_VERSION = "0.80.7";
|
||||
// Clients older than this keep the last catalog published under the previous minimum.
|
||||
// 1.2.1: kimi-coding models use openai-completions at /coding/v1; older clients only
|
||||
// implement anthropic-messages for kimi-coding and ignore a model's api.
|
||||
const MINIMUM_PI_VERSION = "1.2.1";
|
||||
const JSON_CONTENT_TYPE = "application/json; charset=utf-8";
|
||||
const IMMUTABLE_CACHE_CONTROL = "public, max-age=31536000, immutable";
|
||||
const INDEX_CACHE_CONTROL = "no-store";
|
||||
const REQUIRED_PROVIDERS = ["anthropic", "openai", "openrouter"];
|
||||
const MINIMUM_MODEL_COUNT = 500;
|
||||
const MODEL_TYPES = ["chat", "image", "classifier"];
|
||||
|
||||
function parseArgs(args) {
|
||||
const options = {
|
||||
@@ -65,55 +77,81 @@ function readJson(path) {
|
||||
return JSON.parse(readFileSync(path, "utf8"));
|
||||
}
|
||||
|
||||
function isObject(value) {
|
||||
return typeof value === "object" && value !== null && !Array.isArray(value);
|
||||
}
|
||||
|
||||
function chatProjection(providerModels) {
|
||||
return Object.fromEntries(providerModels.filter((model) => model.type === "chat").map((model) => [model.id, model]));
|
||||
}
|
||||
|
||||
function validateBundle(inputDir) {
|
||||
const modelsPath = join(inputDir, "models.json");
|
||||
const allModelsPath = join(inputDir, "models.all.json");
|
||||
const providerIndexPath = join(inputDir, "providers.json");
|
||||
const providersDir = join(inputDir, "providers");
|
||||
const modelsBytes = readFileSync(modelsPath);
|
||||
const allModelsBytes = readFileSync(allModelsPath);
|
||||
const models = JSON.parse(modelsBytes.toString("utf8"));
|
||||
const allModels = JSON.parse(allModelsBytes.toString("utf8"));
|
||||
const providerIds = readJson(providerIndexPath);
|
||||
|
||||
if (typeof models !== "object" || models === null || Array.isArray(models)) {
|
||||
throw new Error("models.json must contain an object");
|
||||
}
|
||||
if (!isObject(models)) throw new Error("models.json must contain an object");
|
||||
if (!isObject(allModels)) throw new Error("models.all.json must contain an object");
|
||||
if (!Array.isArray(providerIds) || !providerIds.every((value) => typeof value === "string")) {
|
||||
throw new Error("providers.json must contain an array of provider IDs");
|
||||
}
|
||||
|
||||
const expectedProviderIds = Object.keys(models).sort();
|
||||
const expectedProviderIds = Object.keys(allModels).sort();
|
||||
if (!isDeepStrictEqual(providerIds, expectedProviderIds)) {
|
||||
throw new Error("providers.json does not match the sorted providers in models.json");
|
||||
throw new Error("providers.json does not match the sorted providers in models.all.json");
|
||||
}
|
||||
if (!isDeepStrictEqual(Object.keys(models).sort(), expectedProviderIds)) {
|
||||
throw new Error("models.json and models.all.json list different providers");
|
||||
}
|
||||
for (const providerId of REQUIRED_PROVIDERS) {
|
||||
if (!Object.hasOwn(models, providerId)) throw new Error(`Required provider is missing: ${providerId}`);
|
||||
}
|
||||
|
||||
// modelCount remains the legacy chat-catalog count for existing consumers.
|
||||
let modelCount = 0;
|
||||
let imageModelCount = 0;
|
||||
let classifierModelCount = 0;
|
||||
for (const providerId of providerIds) {
|
||||
const providerModels = models[providerId];
|
||||
if (typeof providerModels !== "object" || providerModels === null || Array.isArray(providerModels)) {
|
||||
throw new Error(`Provider catalog must be an object: ${providerId}`);
|
||||
}
|
||||
const providerFile = readJson(join(providersDir, `${providerId}.json`));
|
||||
const providerModels = allModels[providerId];
|
||||
if (!Array.isArray(providerModels)) throw new Error(`Full provider catalog must be an array: ${providerId}`);
|
||||
const providerFile = readJson(join(providersDir, `${providerId}.all.json`));
|
||||
if (!isDeepStrictEqual(providerFile, providerModels)) {
|
||||
throw new Error(`Provider shard does not match models.json: ${providerId}`);
|
||||
throw new Error(`Provider shard does not match models.all.json: ${providerId}`);
|
||||
}
|
||||
for (const [modelId, model] of Object.entries(providerModels)) {
|
||||
if (
|
||||
typeof model !== "object" ||
|
||||
model === null ||
|
||||
Array.isArray(model) ||
|
||||
model.id !== modelId ||
|
||||
model.provider !== providerId
|
||||
) {
|
||||
throw new Error(`Invalid model entry: ${providerId}/${modelId}`);
|
||||
const identities = new Set();
|
||||
for (const model of providerModels) {
|
||||
if (!isObject(model) || typeof model.id !== "string" || model.provider !== providerId) {
|
||||
throw new Error(`Invalid model entry in provider catalog: ${providerId}`);
|
||||
}
|
||||
modelCount++;
|
||||
if (!MODEL_TYPES.includes(model.type)) {
|
||||
throw new Error(`Model entry has an unknown type: ${providerId}/${model.id} (${JSON.stringify(model.type)})`);
|
||||
}
|
||||
const identity = `${model.type}:${model.id}`;
|
||||
if (identities.has(identity)) throw new Error(`Duplicate model entry: ${providerId}/${identity}`);
|
||||
identities.add(identity);
|
||||
if (model.type === "chat") modelCount++;
|
||||
else if (model.type === "image") imageModelCount++;
|
||||
else classifierModelCount++;
|
||||
}
|
||||
|
||||
// The legacy variant must be exactly the chat projection of the full catalog.
|
||||
const chatModels = chatProjection(providerModels);
|
||||
if (!isDeepStrictEqual(models[providerId], chatModels)) {
|
||||
throw new Error(`models.json is not the chat projection of models.all.json: ${providerId}`);
|
||||
}
|
||||
if (!isDeepStrictEqual(readJson(join(providersDir, `${providerId}.json`)), chatModels)) {
|
||||
throw new Error(`Provider shard does not match models.json: ${providerId}`);
|
||||
}
|
||||
}
|
||||
|
||||
const shardFiles = readdirSync(providersDir).filter((name) => name.endsWith(".json")).sort();
|
||||
const expectedShardFiles = providerIds.map((providerId) => `${providerId}.json`).sort();
|
||||
const expectedShardFiles = providerIds.flatMap((providerId) => [`${providerId}.json`, `${providerId}.all.json`]).sort();
|
||||
if (!isDeepStrictEqual(shardFiles, expectedShardFiles)) {
|
||||
throw new Error("Provider shard files do not match providers.json");
|
||||
}
|
||||
@@ -121,14 +159,21 @@ function validateBundle(inputDir) {
|
||||
throw new Error(`Refusing to publish only ${modelCount} models; expected at least ${MINIMUM_MODEL_COUNT}`);
|
||||
}
|
||||
|
||||
const digest = createHash("sha256").update(modelsBytes).digest("hex");
|
||||
// The full catalog is a superset of the chat catalog, so hashing it alone
|
||||
// changes the revision for chat-only and image-only updates alike.
|
||||
const digest = createHash("sha256").update(allModelsBytes).digest("hex");
|
||||
return {
|
||||
modelsPath,
|
||||
allModelsPath,
|
||||
providerIndexPath,
|
||||
providersDir,
|
||||
providerIds,
|
||||
providerCount: providerIds.length,
|
||||
modelCount,
|
||||
chatModelCount: modelCount,
|
||||
imageModelCount,
|
||||
classifierModelCount,
|
||||
totalModelCount: modelCount + imageModelCount + classifierModelCount,
|
||||
revision: `sha256-${digest}`,
|
||||
};
|
||||
}
|
||||
@@ -160,7 +205,7 @@ function downloadIndex(bucket, endpoint, outputPath) {
|
||||
[
|
||||
"s3",
|
||||
"cp",
|
||||
`s3://${bucket}/${CATALOG_INDEX_KEY}`,
|
||||
`s3://${bucket}/${MODEL_CATALOG_INDEX_KEY}`,
|
||||
outputPath,
|
||||
"--endpoint-url",
|
||||
endpoint,
|
||||
@@ -186,40 +231,13 @@ function uploadJson(bucket, endpoint, sourcePath, key, cacheControl) {
|
||||
]);
|
||||
}
|
||||
|
||||
// Validate with the same parser pi.dev uses, but keep the stored entries so
|
||||
// the publication metadata of existing revisions is preserved.
|
||||
function validateIndex(index) {
|
||||
if (
|
||||
typeof index !== "object" ||
|
||||
index === null ||
|
||||
Array.isArray(index) ||
|
||||
index.schemaVersion !== CATALOG_SCHEMA_VERSION
|
||||
) {
|
||||
throw new Error(`Existing ${CATALOG_INDEX_KEY} has an unsupported schema`);
|
||||
}
|
||||
if (!Array.isArray(index.catalogs)) throw new Error(`Existing ${CATALOG_INDEX_KEY} has no catalogs array`);
|
||||
for (const catalog of index.catalogs) {
|
||||
if (
|
||||
typeof catalog !== "object" ||
|
||||
catalog === null ||
|
||||
Array.isArray(catalog) ||
|
||||
typeof catalog.minimumPiVersion !== "string" ||
|
||||
typeof catalog.revision !== "string"
|
||||
) {
|
||||
throw new Error(`Existing ${CATALOG_INDEX_KEY} contains an invalid catalog entry`);
|
||||
}
|
||||
}
|
||||
parseModelCatalogIndex(index);
|
||||
return index;
|
||||
}
|
||||
|
||||
function comparePiVersions(left, right) {
|
||||
const leftParts = left.split(".").map(Number);
|
||||
const rightParts = right.split(".").map(Number);
|
||||
for (let index = 0; index < Math.max(leftParts.length, rightParts.length); index++) {
|
||||
const difference = (leftParts[index] || 0) - (rightParts[index] || 0);
|
||||
if (difference !== 0) return difference;
|
||||
}
|
||||
return left.localeCompare(right);
|
||||
}
|
||||
|
||||
function buildIndex(existingIndex, publication) {
|
||||
const entry = {
|
||||
minimumPiVersion: MINIMUM_PI_VERSION,
|
||||
@@ -228,16 +246,21 @@ function buildIndex(existingIndex, publication) {
|
||||
publishedAt: new Date().toISOString(),
|
||||
providerCount: publication.providerCount,
|
||||
modelCount: publication.modelCount,
|
||||
chatModelCount: publication.chatModelCount,
|
||||
imageModelCount: publication.imageModelCount,
|
||||
classifierModelCount: publication.classifierModelCount,
|
||||
totalModelCount: publication.totalModelCount,
|
||||
modelTypes: publication.modelTypes,
|
||||
};
|
||||
const catalogs = (existingIndex?.catalogs || [])
|
||||
.filter((catalog) => catalog.minimumPiVersion !== MINIMUM_PI_VERSION)
|
||||
.concat(entry)
|
||||
.sort((left, right) => comparePiVersions(left.minimumPiVersion, right.minimumPiVersion));
|
||||
return {
|
||||
schemaVersion: CATALOG_SCHEMA_VERSION,
|
||||
.sort((left, right) => compareModelCatalogPiVersions(left.minimumPiVersion, right.minimumPiVersion));
|
||||
return validateIndex({
|
||||
schemaVersion: MODEL_CATALOG_SCHEMA_VERSION,
|
||||
defaultRevision: publication.revision,
|
||||
catalogs,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function main() {
|
||||
@@ -245,12 +268,19 @@ async function main() {
|
||||
const inputDir = resolve(options.input);
|
||||
const bundle = validateBundle(inputDir);
|
||||
const publication = {
|
||||
schemaVersion: CATALOG_SCHEMA_VERSION,
|
||||
schemaVersion: MODEL_CATALOG_SCHEMA_VERSION,
|
||||
minimumPiVersion: MINIMUM_PI_VERSION,
|
||||
revision: bundle.revision,
|
||||
sourceCommit: options.sourceCommit || gitSourceCommit(),
|
||||
providerCount: bundle.providerCount,
|
||||
/** Legacy count for the chat-only `models.json` catalog. */
|
||||
modelCount: bundle.modelCount,
|
||||
chatModelCount: bundle.chatModelCount,
|
||||
imageModelCount: bundle.imageModelCount,
|
||||
classifierModelCount: bundle.classifierModelCount,
|
||||
totalModelCount: bundle.totalModelCount,
|
||||
/** Types present in the `.all` variant of this revision. */
|
||||
modelTypes: MODEL_TYPES,
|
||||
};
|
||||
writeFileSync(join(inputDir, "publication.json"), `${JSON.stringify(publication, null, 2)}\n`);
|
||||
|
||||
@@ -273,30 +303,25 @@ async function main() {
|
||||
return;
|
||||
}
|
||||
|
||||
const revisionPrefix = `${CATALOG_PREFIX}/revisions/${bundle.revision}`;
|
||||
uploadJson(options.bucket, options.endpoint, bundle.modelsPath, `${revisionPrefix}/models.json`, IMMUTABLE_CACHE_CONTROL);
|
||||
uploadJson(
|
||||
options.bucket,
|
||||
options.endpoint,
|
||||
bundle.providerIndexPath,
|
||||
`${revisionPrefix}/providers.json`,
|
||||
IMMUTABLE_CACHE_CONTROL,
|
||||
);
|
||||
for (const providerId of bundle.providerIds) {
|
||||
uploadJson(
|
||||
options.bucket,
|
||||
options.endpoint,
|
||||
join(bundle.providersDir, `${providerId}.json`),
|
||||
`${revisionPrefix}/providers/${providerId}.json`,
|
||||
IMMUTABLE_CACHE_CONTROL,
|
||||
);
|
||||
const revision = bundle.revision;
|
||||
const uploads = [
|
||||
[bundle.modelsPath, getModelCatalogArtifactKey(revision, "models.json")],
|
||||
[bundle.allModelsPath, getModelCatalogArtifactKey(revision, "models.all.json")],
|
||||
[bundle.providerIndexPath, getModelCatalogArtifactKey(revision, "providers.json")],
|
||||
...bundle.providerIds.flatMap((providerId) => [
|
||||
[join(bundle.providersDir, `${providerId}.json`), getModelCatalogProviderKey(revision, providerId, "legacy")],
|
||||
[join(bundle.providersDir, `${providerId}.all.json`), getModelCatalogProviderKey(revision, providerId, "typed")],
|
||||
]),
|
||||
];
|
||||
for (const [sourcePath, key] of uploads) {
|
||||
uploadJson(options.bucket, options.endpoint, sourcePath, key, IMMUTABLE_CACHE_CONTROL);
|
||||
}
|
||||
|
||||
const nextIndex = buildIndex(currentIndex, publication);
|
||||
const nextIndexPath = join(temporaryDir, "index-next.json");
|
||||
writeFileSync(nextIndexPath, `${JSON.stringify(nextIndex, null, 2)}\n`);
|
||||
uploadJson(options.bucket, options.endpoint, nextIndexPath, CATALOG_INDEX_KEY, INDEX_CACHE_CONTROL);
|
||||
console.log(`Published ${bundle.revision} to s3://${options.bucket}/${revisionPrefix}`);
|
||||
uploadJson(options.bucket, options.endpoint, nextIndexPath, MODEL_CATALOG_INDEX_KEY, INDEX_CACHE_CONTROL);
|
||||
console.log(`Published ${revision} to s3://${options.bucket}/${MODEL_CATALOG_PREFIX}/revisions/${revision}`);
|
||||
} finally {
|
||||
rmSync(temporaryDir, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user