alexion
6081e3e325
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
366 lines
12 KiB
TypeScript
366 lines
12 KiB
TypeScript
/**
|
|
* Runs the official MCP client conformance suite against pi (client.ts) and compares every check with
|
|
* the committed baseline (baseline.json). Fails when a check that passed in the baseline no longer
|
|
* passes, or when a check fails that the baseline does not list. Known failures stay visible in the
|
|
* output. See README.md.
|
|
*/
|
|
|
|
import { spawn } from "node:child_process";
|
|
import {
|
|
chmodSync,
|
|
existsSync,
|
|
mkdirSync,
|
|
mkdtempSync,
|
|
readdirSync,
|
|
readFileSync,
|
|
rmSync,
|
|
writeFileSync,
|
|
} from "node:fs";
|
|
import { tmpdir } from "node:os";
|
|
import { join } from "node:path";
|
|
import { fileURLToPath, pathToFileURL } from "node:url";
|
|
import { parseArgs } from "node:util";
|
|
|
|
/** The pinned upstream suite. Bumping it requires reviewing and regenerating the baseline. */
|
|
const CONFORMANCE_PACKAGE = "@modelcontextprotocol/conformance@0.2.0-alpha.11";
|
|
/** Protocol versions pi negotiates. 2026-07-28 is a different (stateless) protocol pi does not speak. */
|
|
const MODES = ["2025-03-26", "2025-06-18", "2025-11-25"] as const;
|
|
const SCENARIO_TIMEOUT_MS = 30_000;
|
|
/** Checks that are part of the suite's HTTP trace, not assertions. */
|
|
const IGNORED_STATUSES = new Set(["INFO", "SKIPPED"]);
|
|
|
|
const here = fileURLToPath(new URL(".", import.meta.url));
|
|
const baselinePath = join(here, "baseline.json");
|
|
const clientPath = join(here, "client.ts");
|
|
const resolverUrl = pathToFileURL(join(here, "../../src/experimental/source-resolver.ts")).href;
|
|
|
|
type CheckStatus = "pass" | "fail";
|
|
|
|
interface ScenarioResult {
|
|
mode: string;
|
|
scenario: string;
|
|
/** Check id to status. Repeated ids pass only when every instance passes. */
|
|
checks: Map<string, CheckStatus>;
|
|
/** Failure messages by check id, for the report. */
|
|
messages: Map<string, string>;
|
|
}
|
|
|
|
interface BaselineScenario {
|
|
passing: string[];
|
|
failing: string[];
|
|
}
|
|
|
|
interface Baseline {
|
|
conformance: string;
|
|
modes: Record<string, Record<string, BaselineScenario>>;
|
|
}
|
|
|
|
interface RawCheck {
|
|
id?: unknown;
|
|
name?: unknown;
|
|
status?: unknown;
|
|
description?: unknown;
|
|
errorMessage?: unknown;
|
|
}
|
|
|
|
function runCommand(
|
|
command: string,
|
|
args: string[],
|
|
options: { cwd: string; env?: NodeJS.ProcessEnv; timeoutMs?: number },
|
|
): Promise<{ code: number; stdout: string; stderr: string }> {
|
|
return new Promise((resolve, reject) => {
|
|
const child = spawn(command, args, { cwd: options.cwd, env: options.env ?? process.env, detached: true });
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdout.on("data", (data: Buffer) => {
|
|
stdout += data.toString();
|
|
});
|
|
child.stderr.on("data", (data: Buffer) => {
|
|
stderr += data.toString();
|
|
});
|
|
const timer =
|
|
options.timeoutMs === undefined
|
|
? undefined
|
|
: setTimeout(() => {
|
|
// The runner spawns the client and scenario servers; end the whole group.
|
|
if (child.pid !== undefined) {
|
|
try {
|
|
process.kill(-child.pid, "SIGKILL");
|
|
} catch {}
|
|
}
|
|
}, options.timeoutMs);
|
|
child.on("error", reject);
|
|
child.on("close", (code, signal) => {
|
|
if (timer) clearTimeout(timer);
|
|
resolve({ code: code ?? (signal ? 128 : 1), stdout, stderr });
|
|
});
|
|
});
|
|
}
|
|
|
|
function conformanceEnv(): NodeJS.ProcessEnv {
|
|
// The suite comes from npm with prebuilt files; nothing needs install scripts.
|
|
return { ...process.env, npm_config_ignore_scripts: "true", npm_config_yes: "true" };
|
|
}
|
|
|
|
async function listScenarios(mode: string, workDir: string): Promise<string[]> {
|
|
const result = await runCommand("npx", ["--yes", CONFORMANCE_PACKAGE, "list", "--client", "--spec-version", mode], {
|
|
cwd: workDir,
|
|
env: conformanceEnv(),
|
|
});
|
|
if (result.code !== 0) throw new Error(`Listing ${mode} scenarios failed:\n${result.stderr || result.stdout}`);
|
|
return [...result.stdout.matchAll(/^\s+- (\S+)/gm)].map((match) => match[1]);
|
|
}
|
|
|
|
function findFile(dir: string, name: string): string | undefined {
|
|
if (!existsSync(dir)) return undefined;
|
|
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
|
const path = join(dir, entry.name);
|
|
if (entry.isFile() && entry.name === name) return path;
|
|
if (entry.isDirectory()) {
|
|
const found = findFile(path, name);
|
|
if (found) return found;
|
|
}
|
|
}
|
|
return undefined;
|
|
}
|
|
|
|
function record(result: ScenarioResult, id: string, status: CheckStatus, message?: string): void {
|
|
if (result.checks.get(id) !== "fail") result.checks.set(id, status);
|
|
if (status === "fail" && message && !result.messages.has(id)) result.messages.set(id, message);
|
|
}
|
|
|
|
async function runScenario(
|
|
mode: string,
|
|
scenario: string,
|
|
launcher: string,
|
|
workDir: string,
|
|
verbose: boolean,
|
|
): Promise<ScenarioResult> {
|
|
const scenarioDir = join(workDir, mode, scenario.replace(/[^A-Za-z0-9_.-]+/g, "-"));
|
|
mkdirSync(scenarioDir, { recursive: true });
|
|
const reportPath = join(scenarioDir, "pi-client.json");
|
|
const result: ScenarioResult = { mode, scenario, checks: new Map(), messages: new Map() };
|
|
const run = await runCommand(
|
|
"npx",
|
|
[
|
|
"--yes",
|
|
CONFORMANCE_PACKAGE,
|
|
"client",
|
|
"--command",
|
|
launcher,
|
|
"--scenario",
|
|
scenario,
|
|
"--spec-version",
|
|
mode,
|
|
"--timeout",
|
|
String(SCENARIO_TIMEOUT_MS),
|
|
"--output-dir",
|
|
scenarioDir,
|
|
],
|
|
{
|
|
cwd: workDir,
|
|
env: { ...conformanceEnv(), PI_MCP_CONFORMANCE_REPORT: reportPath },
|
|
timeoutMs: SCENARIO_TIMEOUT_MS + 30_000,
|
|
},
|
|
);
|
|
if (verbose) process.stderr.write(run.stderr);
|
|
|
|
const checksPath = findFile(scenarioDir, "checks.json");
|
|
if (!checksPath) {
|
|
record(result, "runner", "fail", `No checks.json (runner exit ${run.code}): ${run.stderr.trim().slice(-2000)}`);
|
|
return result;
|
|
}
|
|
const checks = JSON.parse(readFileSync(checksPath, "utf8")) as RawCheck[];
|
|
for (const check of checks) {
|
|
const status = String(check.status ?? "FAILURE").toUpperCase();
|
|
if (IGNORED_STATUSES.has(status)) continue;
|
|
const id = String(check.id ?? check.name ?? "unnamed");
|
|
const message = String(check.errorMessage ?? check.description ?? "");
|
|
record(result, id, status === "SUCCESS" ? "pass" : "fail", message);
|
|
}
|
|
// Whether pi completed the scenario. Some scenarios expect the client to give up, so this is
|
|
// baselined like any other check.
|
|
let client: { success?: unknown; error?: unknown } = { error: "client.ts wrote no report" };
|
|
if (existsSync(reportPath)) client = JSON.parse(readFileSync(reportPath, "utf8")) as typeof client;
|
|
record(result, "pi-client", client.success === true ? "pass" : "fail", String(client.error ?? ""));
|
|
return result;
|
|
}
|
|
|
|
function loadBaseline(): Baseline | undefined {
|
|
if (!existsSync(baselinePath)) return undefined;
|
|
return JSON.parse(readFileSync(baselinePath, "utf8")) as Baseline;
|
|
}
|
|
|
|
function toBaseline(results: ScenarioResult[]): Baseline {
|
|
const baseline: Baseline = { conformance: CONFORMANCE_PACKAGE, modes: {} };
|
|
for (const result of results) {
|
|
const ids = [...result.checks.keys()].sort();
|
|
baseline.modes[result.mode] ??= {};
|
|
baseline.modes[result.mode][result.scenario] = {
|
|
passing: ids.filter((id) => result.checks.get(id) === "pass"),
|
|
failing: ids.filter((id) => result.checks.get(id) === "fail"),
|
|
};
|
|
}
|
|
return baseline;
|
|
}
|
|
|
|
/** Compare with the baseline. Returns whether there is no regression. */
|
|
function evaluate(results: ScenarioResult[], baseline: Baseline): boolean {
|
|
const regressions: string[] = [];
|
|
const fixed: string[] = [];
|
|
for (const result of results) {
|
|
const expected = baseline.modes[result.mode]?.[result.scenario];
|
|
const label = `${result.mode} ${result.scenario}`;
|
|
for (const [id, status] of result.checks) {
|
|
const message = result.messages.get(id);
|
|
if (status === "fail" && !expected?.failing.includes(id)) {
|
|
regressions.push(`${label}: ${id} fails${message ? `: ${message}` : ""}`);
|
|
}
|
|
if (status === "pass" && expected?.failing.includes(id)) fixed.push(`${label}: ${id}`);
|
|
}
|
|
for (const id of expected?.passing ?? []) {
|
|
if (!result.checks.has(id)) regressions.push(`${label}: ${id} was not reported`);
|
|
}
|
|
}
|
|
for (const [mode, scenarios] of Object.entries(baseline.modes)) {
|
|
for (const scenario of Object.keys(scenarios)) {
|
|
if (!results.some((result) => result.mode === mode && result.scenario === scenario)) {
|
|
regressions.push(`${mode} ${scenario}: scenario did not run`);
|
|
}
|
|
}
|
|
}
|
|
if (fixed.length > 0) {
|
|
console.log(
|
|
`\nFixed since the baseline (run with --update-baseline):\n${fixed.map((line) => ` ${line}`).join("\n")}`,
|
|
);
|
|
}
|
|
if (regressions.length > 0) {
|
|
console.log(`\nRegressions:\n${regressions.map((line) => ` ${line}`).join("\n")}`);
|
|
return false;
|
|
}
|
|
console.log("\nNo regressions against the baseline.");
|
|
return true;
|
|
}
|
|
|
|
function printResults(results: ScenarioResult[]): void {
|
|
for (const mode of MODES) {
|
|
const modeResults = results.filter((result) => result.mode === mode);
|
|
if (modeResults.length === 0) continue;
|
|
console.log(`\n${mode}`);
|
|
for (const result of modeResults) {
|
|
const statuses = [...result.checks.values()];
|
|
const failed = statuses.filter((status) => status === "fail").length;
|
|
const passed = statuses.length - failed;
|
|
console.log(` ${failed === 0 ? "pass" : "FAIL"} ${result.scenario} (${passed} passed, ${failed} failed)`);
|
|
for (const [id, status] of result.checks) {
|
|
if (status !== "fail") continue;
|
|
const message = result.messages.get(id)?.replace(/\s+/g, " ").slice(0, 300);
|
|
console.log(` ${id}${message ? `: ${message}` : ""}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
async function main(): Promise<number> {
|
|
const { values } = parseArgs({
|
|
options: {
|
|
mode: { type: "string", multiple: true },
|
|
scenario: { type: "string", multiple: true },
|
|
"update-baseline": { type: "boolean", default: false },
|
|
"keep-results": { type: "boolean", default: false },
|
|
verbose: { type: "boolean", default: false },
|
|
help: { type: "boolean", short: "h", default: false },
|
|
},
|
|
});
|
|
if (values.help) {
|
|
console.log(`Usage: node packages/coding-agent/test/mcp-conformance/run.ts [options]
|
|
|
|
Options:
|
|
--mode <version> Only run this protocol version (repeatable): ${MODES.join(", ")}
|
|
--scenario <name> Only run this scenario (repeatable)
|
|
--update-baseline Write the results to baseline.json instead of comparing
|
|
--keep-results Keep the upstream checks.json and client output
|
|
--verbose Print the upstream runner output`);
|
|
return 0;
|
|
}
|
|
if (process.platform === "win32") {
|
|
console.error("The conformance runner needs a POSIX shell.");
|
|
return 1;
|
|
}
|
|
const modes = values.mode ?? [...MODES];
|
|
for (const mode of modes) {
|
|
if (!(MODES as readonly string[]).includes(mode)) {
|
|
console.error(`Unsupported mode ${mode}. Supported: ${MODES.join(", ")}`);
|
|
return 1;
|
|
}
|
|
}
|
|
const partial = values.mode !== undefined || values.scenario !== undefined;
|
|
if (values["update-baseline"] && partial) {
|
|
console.error("--update-baseline needs a full run, without --mode or --scenario.");
|
|
return 1;
|
|
}
|
|
|
|
const workDir = mkdtempSync(join(tmpdir(), "pi-mcp-conformance-"));
|
|
try {
|
|
// The upstream runner splits --command on spaces, so use a launcher path without any.
|
|
const launcher = join(workDir, "pi-client");
|
|
writeFileSync(
|
|
launcher,
|
|
`#!/bin/sh\nexec ${JSON.stringify(process.execPath)} --import ${JSON.stringify(resolverUrl)} ${JSON.stringify(clientPath)} "$@"\n`,
|
|
);
|
|
chmodSync(launcher, 0o755);
|
|
|
|
const results: ScenarioResult[] = [];
|
|
for (const mode of modes) {
|
|
const scenarios = (await listScenarios(mode, workDir)).filter(
|
|
(scenario) => !values.scenario || values.scenario.includes(scenario),
|
|
);
|
|
for (const scenario of scenarios) {
|
|
process.stderr.write(`${mode} ${scenario}\n`);
|
|
results.push(await runScenario(mode, scenario, launcher, workDir, values.verbose));
|
|
}
|
|
}
|
|
if (results.length === 0) {
|
|
console.error("No scenarios matched.");
|
|
return 1;
|
|
}
|
|
printResults(results);
|
|
|
|
if (values["update-baseline"]) {
|
|
writeFileSync(baselinePath, `${JSON.stringify(toBaseline(results), null, "\t")}\n`);
|
|
console.log(`\nWrote ${baselinePath}`);
|
|
return 0;
|
|
}
|
|
const baseline = loadBaseline();
|
|
if (!baseline) {
|
|
console.error("\nNo baseline.json. Run with --update-baseline and review it.");
|
|
return 1;
|
|
}
|
|
if (baseline.conformance !== CONFORMANCE_PACKAGE) {
|
|
console.error(`\nbaseline.json is for ${baseline.conformance}, not ${CONFORMANCE_PACKAGE}. Regenerate it.`);
|
|
return 1;
|
|
}
|
|
const scoped: Baseline = partial
|
|
? {
|
|
conformance: baseline.conformance,
|
|
modes: Object.fromEntries(
|
|
modes.map((mode) => [
|
|
mode,
|
|
Object.fromEntries(
|
|
Object.entries(baseline.modes[mode] ?? {}).filter(
|
|
([scenario]) => !values.scenario || values.scenario.includes(scenario),
|
|
),
|
|
),
|
|
]),
|
|
),
|
|
}
|
|
: baseline;
|
|
return evaluate(results, scoped) ? 0 : 1;
|
|
} finally {
|
|
if (values["keep-results"]) console.log(`\nResults kept in ${workDir}`);
|
|
else rmSync(workDir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
process.exit(await main());
|