chore(vendor): import pi 1.3.0
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
40
third_party/pi/packages/evals/evals/models.docs.eval.ts
vendored
Normal file
40
third_party/pi/packages/evals/evals/models.docs.eval.ts
vendored
Normal file
@@ -0,0 +1,40 @@
|
||||
import { describeEval, StructuredOutputJudge } from "vitest-evals";
|
||||
import { inspectAddedModel, type AddedModelOutput } from "./configured-runtime.ts";
|
||||
import { createPiDocumentationEvalHarness } from "../src/harness.ts";
|
||||
|
||||
const PROVIDER_ID = "openai";
|
||||
const MODEL_ID = "fixture-chat";
|
||||
const MODEL_NAME = "Fixture Chat";
|
||||
|
||||
const expected: AddedModelOutput = {
|
||||
result: {
|
||||
model: {
|
||||
id: MODEL_ID,
|
||||
name: MODEL_NAME,
|
||||
provider: PROVIDER_ID,
|
||||
reasoning: true,
|
||||
input: ["text"],
|
||||
cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
|
||||
contextWindow: 32768,
|
||||
maxTokens: 4096,
|
||||
},
|
||||
existingModelsPreserved: true,
|
||||
},
|
||||
};
|
||||
|
||||
const harness = createPiDocumentationEvalHarness({
|
||||
output: ({ session }) => inspectAddedModel(session.modelRuntime, PROVIDER_ID, MODEL_ID),
|
||||
});
|
||||
const judge = StructuredOutputJudge({ expected, match: "strict", allowExtras: false });
|
||||
|
||||
describeEval("Add model to existing provider", { harness, judges: [judge], judgeThreshold: null }, (it) => {
|
||||
it("adds the model without replacing existing models", async ({ run }) => {
|
||||
await run([
|
||||
{
|
||||
type: "prompt",
|
||||
content: `Configure this running Pi installation with a new \`${PROVIDER_ID}/${MODEL_ID}\` model. Do not create project-local configuration. Show it as “${MODEL_NAME}”. It accepts text, supports reasoning, has a 32,768-token context window and a 4,096-token maximum output, and has no usage cost.`,
|
||||
},
|
||||
{ type: "reload" },
|
||||
]);
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user