chore(vendor): import pi 1.3.0
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
This commit is contained in:
45
third_party/pi/packages/coding-agent/test/mcp-conformance/README.md
vendored
Normal file
45
third_party/pi/packages/coding-agent/test/mcp-conformance/README.md
vendored
Normal file
@@ -0,0 +1,45 @@
|
||||
# MCP client conformance
|
||||
|
||||
Runs the official [MCP conformance suite](https://github.com/modelcontextprotocol/conformance) against pi's MCP client and compares the result with a committed baseline.
|
||||
|
||||
```bash
|
||||
npm run test:mcp-conformance
|
||||
```
|
||||
|
||||
CI runs it in the `mcp-conformance` job of `.github/workflows/ci.yml`. Requires network access the first time, to fetch the pinned suite (`@modelcontextprotocol/conformance@0.2.0-alpha.11`) with `npx`. Install scripts are disabled. Needs a POSIX shell.
|
||||
|
||||
## How it works
|
||||
|
||||
For every protocol version pi negotiates (`2025-03-26`, `2025-06-18`, `2025-11-25`), `run.ts` lists the suite's client scenarios and runs them one at a time. The suite starts a scenario server and runs `client.ts` against it.
|
||||
|
||||
`client.ts` uses the code pi runs for an HTTP server in `mcp.json`: `McpServerConnection` connects and calls tools, and `signInMcpServer` runs the OAuth sign-in that `/mcp` starts. A simulated browser fetches the authorization URL and delivers the redirect to pi's loopback callback server. When the server asks for sign-in again (for example for more scope), the simulated user signs in again, up to three times.
|
||||
|
||||
Every check the suite reports is compared with `baseline.json`. The extra `pi-client` check records whether `client.ts` completed the scenario. Some scenarios expect the client to give up (`auth/scope-retry-limit`), so it is baselined like the others.
|
||||
|
||||
The run fails when:
|
||||
|
||||
- a check that passes in the baseline fails or is missing,
|
||||
- a check fails that the baseline does not list as failing,
|
||||
- a baselined scenario did not run.
|
||||
|
||||
Checks that started passing are reported; update the baseline to lock them in.
|
||||
|
||||
## Options
|
||||
|
||||
```bash
|
||||
node packages/coding-agent/test/mcp-conformance/run.ts --mode 2025-11-25 --scenario auth/scope-step-up --verbose
|
||||
node packages/coding-agent/test/mcp-conformance/run.ts --update-baseline
|
||||
```
|
||||
|
||||
- `--mode`, `--scenario`: run a subset (repeatable).
|
||||
- `--verbose`: print the suite's output, including each check and the client's log.
|
||||
- `--keep-results`: keep the suite's `checks.json` and client output.
|
||||
- `--update-baseline`: write the results of a full run to `baseline.json`. Review the diff; do not regenerate it to hide a regression.
|
||||
|
||||
## Known failures
|
||||
|
||||
- `elicitation-sep1034-client-defaults`: pi does not support elicitation.
|
||||
- `auth/basic-cimd`: pi registers clients dynamically instead of using a Client ID Metadata Document.
|
||||
- `auth/scope-retry-limit`: `pi-client` fails by design; the server never accepts the granted scope.
|
||||
|
||||
2026-07-28 is not covered: it is a stateless protocol pi does not implement.
|
||||
413
third_party/pi/packages/coding-agent/test/mcp-conformance/baseline.json
vendored
Normal file
413
third_party/pi/packages/coding-agent/test/mcp-conformance/baseline.json
vendored
Normal file
@@ -0,0 +1,413 @@
|
||||
{
|
||||
"conformance": "@modelcontextprotocol/conformance@0.2.0-alpha.11",
|
||||
"modes": {
|
||||
"2025-03-26": {
|
||||
"auth/2025-03-26-oauth-metadata-backcompat": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/2025-03-26-oauth-endpoint-fallback": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
}
|
||||
},
|
||||
"2025-06-18": {
|
||||
"initialize": {
|
||||
"passing": [
|
||||
"mcp-client-initialization",
|
||||
"pi-client"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"tools_call": {
|
||||
"passing": [
|
||||
"pi-client",
|
||||
"tool-add-numbers",
|
||||
"wire-schema-valid"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/token-endpoint-auth-basic": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"resource-parameter-consistency",
|
||||
"resource-parameter-in-authorization",
|
||||
"resource-parameter-in-token",
|
||||
"resource-parameter-valid-uri",
|
||||
"token-endpoint-auth-method",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/token-endpoint-auth-post": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"resource-parameter-consistency",
|
||||
"resource-parameter-in-authorization",
|
||||
"resource-parameter-in-token",
|
||||
"resource-parameter-valid-uri",
|
||||
"token-endpoint-auth-method",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/token-endpoint-auth-none": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"resource-parameter-consistency",
|
||||
"resource-parameter-in-authorization",
|
||||
"resource-parameter-in-token",
|
||||
"resource-parameter-valid-uri",
|
||||
"token-endpoint-auth-method",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
}
|
||||
},
|
||||
"2025-11-25": {
|
||||
"initialize": {
|
||||
"passing": [
|
||||
"mcp-client-initialization",
|
||||
"pi-client"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"tools_call": {
|
||||
"passing": [
|
||||
"pi-client",
|
||||
"tool-add-numbers",
|
||||
"wire-schema-valid"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"elicitation-sep1034-client-defaults": {
|
||||
"passing": [
|
||||
"pi-client"
|
||||
],
|
||||
"failing": [
|
||||
"client-elicitation-sep1034-boolean-default",
|
||||
"client-elicitation-sep1034-enum-default",
|
||||
"client-elicitation-sep1034-general",
|
||||
"client-elicitation-sep1034-integer-default",
|
||||
"client-elicitation-sep1034-number-default",
|
||||
"client-elicitation-sep1034-string-default"
|
||||
]
|
||||
},
|
||||
"sse-retry": {
|
||||
"passing": [
|
||||
"client-sse-graceful-reconnect",
|
||||
"client-sse-last-event-id",
|
||||
"client-sse-retry-timing",
|
||||
"pi-client"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/metadata-default": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/metadata-var1": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/metadata-var2": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/metadata-var3": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/basic-cimd": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": [
|
||||
"cimd-client-id-used"
|
||||
]
|
||||
},
|
||||
"auth/scope-from-www-authenticate": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"scope-from-www-authenticate",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/scope-from-scopes-supported": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"scope-from-scopes-supported",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/scope-omitted-when-undefined": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"scope-omitted-when-undefined",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/scope-step-up": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"scope-step-up-escalation",
|
||||
"scope-step-up-initial",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/scope-retry-limit": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"scope-retry-limit",
|
||||
"token-request"
|
||||
],
|
||||
"failing": [
|
||||
"pi-client"
|
||||
]
|
||||
},
|
||||
"auth/token-endpoint-auth-basic": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"resource-parameter-consistency",
|
||||
"resource-parameter-in-authorization",
|
||||
"resource-parameter-in-token",
|
||||
"resource-parameter-valid-uri",
|
||||
"token-endpoint-auth-method",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/token-endpoint-auth-post": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"resource-parameter-consistency",
|
||||
"resource-parameter-in-authorization",
|
||||
"resource-parameter-in-token",
|
||||
"resource-parameter-valid-uri",
|
||||
"token-endpoint-auth-method",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/token-endpoint-auth-none": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"client-registration",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"prm-pathbased-requested",
|
||||
"resource-parameter-consistency",
|
||||
"resource-parameter-in-authorization",
|
||||
"resource-parameter-in-token",
|
||||
"resource-parameter-valid-uri",
|
||||
"token-endpoint-auth-method",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"auth/pre-registration": {
|
||||
"passing": [
|
||||
"authorization-request",
|
||||
"authorization-server-metadata",
|
||||
"pi-client",
|
||||
"pkce-code-challenge-sent",
|
||||
"pkce-code-verifier-sent",
|
||||
"pkce-s256-method-used",
|
||||
"pkce-verifier-matches-challenge",
|
||||
"pre-registration-auth",
|
||||
"prm-pathbased-requested",
|
||||
"token-request",
|
||||
"valid-bearer-token"
|
||||
],
|
||||
"failing": []
|
||||
},
|
||||
"json-schema-2020-12-preservation": {
|
||||
"passing": [
|
||||
"json-schema-2020-12-client-$defs-preserved",
|
||||
"json-schema-2020-12-client-$schema-preserved",
|
||||
"json-schema-2020-12-client-additionalProperties-preserved",
|
||||
"json-schema-2020-12-client-echo-completed",
|
||||
"json-schema-2020-12-client-tool-found",
|
||||
"pi-client",
|
||||
"sep-2106-client-anchor-keyword-preserved",
|
||||
"sep-2106-client-composition-keywords-preserved",
|
||||
"sep-2106-client-conditional-keywords-preserved",
|
||||
"wire-schema-valid"
|
||||
],
|
||||
"failing": []
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
200
third_party/pi/packages/coding-agent/test/mcp-conformance/client.ts
vendored
Normal file
200
third_party/pi/packages/coding-agent/test/mcp-conformance/client.ts
vendored
Normal file
@@ -0,0 +1,200 @@
|
||||
/**
|
||||
* Client under test for the official MCP client conformance suite. The suite starts a scenario server
|
||||
* and runs this script with the server URL as the last argument, the scenario name in
|
||||
* `MCP_CONFORMANCE_SCENARIO`, and scenario data in `MCP_CONFORMANCE_CONTEXT`.
|
||||
*
|
||||
* It drives the code pi runs for an `mcp.json` HTTP server: `McpServerConnection` connects and calls
|
||||
* tools, and `signInMcpServer` runs the OAuth sign-in that `/mcp` starts. The browser is simulated: the
|
||||
* authorization URL is fetched without following its redirect, and the redirect is delivered to pi's
|
||||
* loopback callback server. Credentials stay in memory.
|
||||
*
|
||||
* Run through run.ts, which writes the outcome to `PI_MCP_CONFORMANCE_REPORT`.
|
||||
*/
|
||||
|
||||
import { writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { InMemoryAuthStorageBackend } from "../../src/core/auth-storage.ts";
|
||||
import type { McpHttpServerConfig, McpServerEntry } from "../../src/extensions/mcp/config.ts";
|
||||
import type { McpSignInPrompt } from "../../src/extensions/mcp/oauth.ts";
|
||||
import {
|
||||
createDefaultTransport,
|
||||
McpOAuthCredentialStore,
|
||||
McpServerConnection,
|
||||
signInMcpServer,
|
||||
} from "../../src/extensions/mcp/runtime.ts";
|
||||
|
||||
/**
|
||||
* Sign-ins pi asks the user for are not limited, so a user who keeps approving them would loop
|
||||
* forever against a server that never accepts the granted scope. This simulated user gives up after
|
||||
* three, the limit `auth/scope-retry-limit` checks.
|
||||
*/
|
||||
const MAX_SIGN_INS = 3;
|
||||
const REQUEST_TIMEOUT_SECONDS = 20;
|
||||
|
||||
interface ToolCall {
|
||||
name: string;
|
||||
arguments: Record<string, unknown>;
|
||||
}
|
||||
|
||||
/** Tool calls of a scenario after connecting. `undefined` for scenarios this client does not know. */
|
||||
function toolCalls(scenario: string, connection: McpServerConnection): ToolCall[] | undefined {
|
||||
if (scenario.startsWith("auth/")) return [{ name: "test-tool", arguments: {} }];
|
||||
switch (scenario) {
|
||||
case "initialize":
|
||||
return [];
|
||||
case "tools_call":
|
||||
return [{ name: "add_numbers", arguments: { a: 2, b: 3 } }];
|
||||
case "sse-retry":
|
||||
return [{ name: "test_reconnection", arguments: {} }];
|
||||
case "elicitation-sep1034-client-defaults":
|
||||
return [{ name: "test_client_elicitation_defaults", arguments: {} }];
|
||||
case "json-schema-2020-12-preservation": {
|
||||
// The server compares the echoed schema with the one it listed, to detect dropped keywords.
|
||||
const tool = connection.tools.find((candidate) => candidate.name === "json_schema_2020_12_tool");
|
||||
if (!tool) throw new Error("json_schema_2020_12_tool was not listed");
|
||||
return [{ name: "json_schema_echo", arguments: { schema: tool.inputSchema } }];
|
||||
}
|
||||
default:
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
function log(message: string): void {
|
||||
process.stderr.write(`[pi-conformance] ${message}\n`);
|
||||
}
|
||||
|
||||
function errorMessage(error: unknown): string {
|
||||
return error instanceof Error ? error.message : String(error);
|
||||
}
|
||||
|
||||
function readContext(): Record<string, unknown> {
|
||||
const raw = process.env.MCP_CONFORMANCE_CONTEXT;
|
||||
if (!raw) return {};
|
||||
const parsed: unknown = JSON.parse(raw);
|
||||
if (typeof parsed !== "object" || parsed === null || Array.isArray(parsed)) {
|
||||
throw new Error("MCP_CONFORMANCE_CONTEXT is not an object");
|
||||
}
|
||||
return parsed as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function isLoopback(url: URL): boolean {
|
||||
return ["localhost", "127.0.0.1", "[::1]"].includes(url.hostname);
|
||||
}
|
||||
|
||||
/** What a browser does with the authorization URL when the user approves at once. */
|
||||
async function visitAuthorizationUrl(url: URL): Promise<void> {
|
||||
const response = await fetch(url, { redirect: "manual" });
|
||||
const location = response.headers.get("location");
|
||||
if (!location) throw new Error(`Authorization endpoint answered ${response.status} without a redirect`);
|
||||
const callback = new URL(location, url);
|
||||
if (callback.protocol !== "http:" || !isLoopback(callback)) {
|
||||
throw new Error(`Authorization endpoint redirected to ${callback.origin}, not to the loopback callback`);
|
||||
}
|
||||
log(`delivering authorization response to ${callback.origin}${callback.pathname}`);
|
||||
await (await fetch(callback)).text();
|
||||
}
|
||||
|
||||
function simulatedBrowser(): McpSignInPrompt {
|
||||
let visit: Promise<void> | undefined;
|
||||
return {
|
||||
showAuthorizationUrl(url) {
|
||||
log(`authorization URL: ${url.origin}${url.pathname}`);
|
||||
visit = visitAuthorizationUrl(url);
|
||||
},
|
||||
async promptForRedirectUrl(signal) {
|
||||
try {
|
||||
await visit;
|
||||
} catch (error) {
|
||||
// Cancels the sign-in instead of waiting for a callback that never comes.
|
||||
log(`browser failed: ${errorMessage(error)}`);
|
||||
return undefined;
|
||||
}
|
||||
if (!signal.aborted) {
|
||||
await new Promise((resolve) => signal.addEventListener("abort", resolve, { once: true }));
|
||||
}
|
||||
return undefined;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
async function run(serverUrl: string, scenario: string): Promise<void> {
|
||||
const context = readContext();
|
||||
const config: McpHttpServerConfig = { url: serverUrl, timeout: REQUEST_TIMEOUT_SECONDS };
|
||||
if (typeof context.client_id === "string") {
|
||||
// Pre-registered clients are configured in `mcp.json`.
|
||||
config.oauth = {
|
||||
clientId: context.client_id,
|
||||
clientSecret: typeof context.client_secret === "string" ? context.client_secret : undefined,
|
||||
};
|
||||
}
|
||||
const entry: McpServerEntry = { name: "conformance", config, source: "conformance" };
|
||||
const credentials = new McpOAuthCredentialStore(new InMemoryAuthStorageBackend());
|
||||
const connection = new McpServerConnection({
|
||||
entry,
|
||||
cwd: tmpdir(),
|
||||
createTransport: createDefaultTransport,
|
||||
credentials,
|
||||
onTools: () => {},
|
||||
});
|
||||
|
||||
let signIns = 0;
|
||||
/** Run `operation`, signing in like a user answering `/mcp` whenever the server asks for it. */
|
||||
const withSignIn = async <T>(operation: () => Promise<T>): Promise<T> => {
|
||||
for (;;) {
|
||||
try {
|
||||
return await operation();
|
||||
} catch (error) {
|
||||
const url = connection.oauthUrl;
|
||||
if (connection.state !== "needs-auth" || !url) throw error;
|
||||
if (signIns >= MAX_SIGN_INS) throw new Error(`Still requires sign-in after ${signIns} sign-ins`);
|
||||
signIns++;
|
||||
log(`sign-in ${signIns}${connection.challenge?.scope ? ` (scope: ${connection.challenge.scope})` : ""}`);
|
||||
await signInMcpServer({
|
||||
serverUrl: url,
|
||||
store: credentials.forServer(entry.name, url),
|
||||
settings: connection.oauthSettings(),
|
||||
challenge: connection.challenge,
|
||||
prompt: simulatedBrowser(),
|
||||
});
|
||||
connection.challenge = undefined;
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
await withSignIn(() => connection.getClient());
|
||||
log(`connected, tools: ${connection.tools.map((tool) => tool.name).join(", ") || "(none)"}`);
|
||||
const calls = toolCalls(scenario, connection);
|
||||
if (!calls) throw new Error(`Unknown scenario ${scenario}`);
|
||||
for (const call of calls) {
|
||||
const result = await withSignIn(() => connection.callTool(call.name, call.arguments, {}));
|
||||
log(`${call.name}: ${JSON.stringify(result.content)}`);
|
||||
if (result.isError) throw new Error(`Tool ${call.name} failed: ${JSON.stringify(result.content)}`);
|
||||
}
|
||||
} finally {
|
||||
await connection.close();
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const serverUrl = process.argv.at(-1);
|
||||
const scenario = process.env.MCP_CONFORMANCE_SCENARIO;
|
||||
const reportPath = process.env.PI_MCP_CONFORMANCE_REPORT;
|
||||
let report: { success: boolean; error?: string };
|
||||
if (!serverUrl || !scenario || process.argv.length < 3) {
|
||||
report = { success: false, error: "Usage: MCP_CONFORMANCE_SCENARIO=<scenario> client.ts <server-url>" };
|
||||
} else {
|
||||
try {
|
||||
await run(serverUrl, scenario);
|
||||
report = { success: true };
|
||||
} catch (error) {
|
||||
report = { success: false, error: errorMessage(error) };
|
||||
}
|
||||
}
|
||||
if (report.error) log(`error: ${report.error}`);
|
||||
if (reportPath) writeFileSync(reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
return report.success ? 0 : 1;
|
||||
}
|
||||
|
||||
// Exit explicitly: a failed scenario can leave sockets of the server under test open.
|
||||
process.exit(await main());
|
||||
365
third_party/pi/packages/coding-agent/test/mcp-conformance/run.ts
vendored
Normal file
365
third_party/pi/packages/coding-agent/test/mcp-conformance/run.ts
vendored
Normal file
@@ -0,0 +1,365 @@
|
||||
/**
|
||||
* Runs the official MCP client conformance suite against pi (client.ts) and compares every check with
|
||||
* the committed baseline (baseline.json). Fails when a check that passed in the baseline no longer
|
||||
* passes, or when a check fails that the baseline does not list. Known failures stay visible in the
|
||||
* output. See README.md.
|
||||
*/
|
||||
|
||||
import { spawn } from "node:child_process";
|
||||
import {
|
||||
chmodSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
mkdtempSync,
|
||||
readdirSync,
|
||||
readFileSync,
|
||||
rmSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
/** The pinned upstream suite. Bumping it requires reviewing and regenerating the baseline. */
|
||||
const CONFORMANCE_PACKAGE = "@modelcontextprotocol/conformance@0.2.0-alpha.11";
|
||||
/** Protocol versions pi negotiates. 2026-07-28 is a different (stateless) protocol pi does not speak. */
|
||||
const MODES = ["2025-03-26", "2025-06-18", "2025-11-25"] as const;
|
||||
const SCENARIO_TIMEOUT_MS = 30_000;
|
||||
/** Checks that are part of the suite's HTTP trace, not assertions. */
|
||||
const IGNORED_STATUSES = new Set(["INFO", "SKIPPED"]);
|
||||
|
||||
const here = fileURLToPath(new URL(".", import.meta.url));
|
||||
const baselinePath = join(here, "baseline.json");
|
||||
const clientPath = join(here, "client.ts");
|
||||
const resolverUrl = pathToFileURL(join(here, "../../src/experimental/source-resolver.ts")).href;
|
||||
|
||||
type CheckStatus = "pass" | "fail";
|
||||
|
||||
interface ScenarioResult {
|
||||
mode: string;
|
||||
scenario: string;
|
||||
/** Check id to status. Repeated ids pass only when every instance passes. */
|
||||
checks: Map<string, CheckStatus>;
|
||||
/** Failure messages by check id, for the report. */
|
||||
messages: Map<string, string>;
|
||||
}
|
||||
|
||||
interface BaselineScenario {
|
||||
passing: string[];
|
||||
failing: string[];
|
||||
}
|
||||
|
||||
interface Baseline {
|
||||
conformance: string;
|
||||
modes: Record<string, Record<string, BaselineScenario>>;
|
||||
}
|
||||
|
||||
interface RawCheck {
|
||||
id?: unknown;
|
||||
name?: unknown;
|
||||
status?: unknown;
|
||||
description?: unknown;
|
||||
errorMessage?: unknown;
|
||||
}
|
||||
|
||||
function runCommand(
|
||||
command: string,
|
||||
args: string[],
|
||||
options: { cwd: string; env?: NodeJS.ProcessEnv; timeoutMs?: number },
|
||||
): Promise<{ code: number; stdout: string; stderr: string }> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const child = spawn(command, args, { cwd: options.cwd, env: options.env ?? process.env, detached: true });
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (data: Buffer) => {
|
||||
stdout += data.toString();
|
||||
});
|
||||
child.stderr.on("data", (data: Buffer) => {
|
||||
stderr += data.toString();
|
||||
});
|
||||
const timer =
|
||||
options.timeoutMs === undefined
|
||||
? undefined
|
||||
: setTimeout(() => {
|
||||
// The runner spawns the client and scenario servers; end the whole group.
|
||||
if (child.pid !== undefined) {
|
||||
try {
|
||||
process.kill(-child.pid, "SIGKILL");
|
||||
} catch {}
|
||||
}
|
||||
}, options.timeoutMs);
|
||||
child.on("error", reject);
|
||||
child.on("close", (code, signal) => {
|
||||
if (timer) clearTimeout(timer);
|
||||
resolve({ code: code ?? (signal ? 128 : 1), stdout, stderr });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function conformanceEnv(): NodeJS.ProcessEnv {
|
||||
// The suite comes from npm with prebuilt files; nothing needs install scripts.
|
||||
return { ...process.env, npm_config_ignore_scripts: "true", npm_config_yes: "true" };
|
||||
}
|
||||
|
||||
async function listScenarios(mode: string, workDir: string): Promise<string[]> {
|
||||
const result = await runCommand("npx", ["--yes", CONFORMANCE_PACKAGE, "list", "--client", "--spec-version", mode], {
|
||||
cwd: workDir,
|
||||
env: conformanceEnv(),
|
||||
});
|
||||
if (result.code !== 0) throw new Error(`Listing ${mode} scenarios failed:\n${result.stderr || result.stdout}`);
|
||||
return [...result.stdout.matchAll(/^\s+- (\S+)/gm)].map((match) => match[1]);
|
||||
}
|
||||
|
||||
function findFile(dir: string, name: string): string | undefined {
|
||||
if (!existsSync(dir)) return undefined;
|
||||
for (const entry of readdirSync(dir, { withFileTypes: true })) {
|
||||
const path = join(dir, entry.name);
|
||||
if (entry.isFile() && entry.name === name) return path;
|
||||
if (entry.isDirectory()) {
|
||||
const found = findFile(path, name);
|
||||
if (found) return found;
|
||||
}
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function record(result: ScenarioResult, id: string, status: CheckStatus, message?: string): void {
|
||||
if (result.checks.get(id) !== "fail") result.checks.set(id, status);
|
||||
if (status === "fail" && message && !result.messages.has(id)) result.messages.set(id, message);
|
||||
}
|
||||
|
||||
async function runScenario(
|
||||
mode: string,
|
||||
scenario: string,
|
||||
launcher: string,
|
||||
workDir: string,
|
||||
verbose: boolean,
|
||||
): Promise<ScenarioResult> {
|
||||
const scenarioDir = join(workDir, mode, scenario.replace(/[^A-Za-z0-9_.-]+/g, "-"));
|
||||
mkdirSync(scenarioDir, { recursive: true });
|
||||
const reportPath = join(scenarioDir, "pi-client.json");
|
||||
const result: ScenarioResult = { mode, scenario, checks: new Map(), messages: new Map() };
|
||||
const run = await runCommand(
|
||||
"npx",
|
||||
[
|
||||
"--yes",
|
||||
CONFORMANCE_PACKAGE,
|
||||
"client",
|
||||
"--command",
|
||||
launcher,
|
||||
"--scenario",
|
||||
scenario,
|
||||
"--spec-version",
|
||||
mode,
|
||||
"--timeout",
|
||||
String(SCENARIO_TIMEOUT_MS),
|
||||
"--output-dir",
|
||||
scenarioDir,
|
||||
],
|
||||
{
|
||||
cwd: workDir,
|
||||
env: { ...conformanceEnv(), PI_MCP_CONFORMANCE_REPORT: reportPath },
|
||||
timeoutMs: SCENARIO_TIMEOUT_MS + 30_000,
|
||||
},
|
||||
);
|
||||
if (verbose) process.stderr.write(run.stderr);
|
||||
|
||||
const checksPath = findFile(scenarioDir, "checks.json");
|
||||
if (!checksPath) {
|
||||
record(result, "runner", "fail", `No checks.json (runner exit ${run.code}): ${run.stderr.trim().slice(-2000)}`);
|
||||
return result;
|
||||
}
|
||||
const checks = JSON.parse(readFileSync(checksPath, "utf8")) as RawCheck[];
|
||||
for (const check of checks) {
|
||||
const status = String(check.status ?? "FAILURE").toUpperCase();
|
||||
if (IGNORED_STATUSES.has(status)) continue;
|
||||
const id = String(check.id ?? check.name ?? "unnamed");
|
||||
const message = String(check.errorMessage ?? check.description ?? "");
|
||||
record(result, id, status === "SUCCESS" ? "pass" : "fail", message);
|
||||
}
|
||||
// Whether pi completed the scenario. Some scenarios expect the client to give up, so this is
|
||||
// baselined like any other check.
|
||||
let client: { success?: unknown; error?: unknown } = { error: "client.ts wrote no report" };
|
||||
if (existsSync(reportPath)) client = JSON.parse(readFileSync(reportPath, "utf8")) as typeof client;
|
||||
record(result, "pi-client", client.success === true ? "pass" : "fail", String(client.error ?? ""));
|
||||
return result;
|
||||
}
|
||||
|
||||
function loadBaseline(): Baseline | undefined {
|
||||
if (!existsSync(baselinePath)) return undefined;
|
||||
return JSON.parse(readFileSync(baselinePath, "utf8")) as Baseline;
|
||||
}
|
||||
|
||||
function toBaseline(results: ScenarioResult[]): Baseline {
|
||||
const baseline: Baseline = { conformance: CONFORMANCE_PACKAGE, modes: {} };
|
||||
for (const result of results) {
|
||||
const ids = [...result.checks.keys()].sort();
|
||||
baseline.modes[result.mode] ??= {};
|
||||
baseline.modes[result.mode][result.scenario] = {
|
||||
passing: ids.filter((id) => result.checks.get(id) === "pass"),
|
||||
failing: ids.filter((id) => result.checks.get(id) === "fail"),
|
||||
};
|
||||
}
|
||||
return baseline;
|
||||
}
|
||||
|
||||
/** Compare with the baseline. Returns whether there is no regression. */
|
||||
function evaluate(results: ScenarioResult[], baseline: Baseline): boolean {
|
||||
const regressions: string[] = [];
|
||||
const fixed: string[] = [];
|
||||
for (const result of results) {
|
||||
const expected = baseline.modes[result.mode]?.[result.scenario];
|
||||
const label = `${result.mode} ${result.scenario}`;
|
||||
for (const [id, status] of result.checks) {
|
||||
const message = result.messages.get(id);
|
||||
if (status === "fail" && !expected?.failing.includes(id)) {
|
||||
regressions.push(`${label}: ${id} fails${message ? `: ${message}` : ""}`);
|
||||
}
|
||||
if (status === "pass" && expected?.failing.includes(id)) fixed.push(`${label}: ${id}`);
|
||||
}
|
||||
for (const id of expected?.passing ?? []) {
|
||||
if (!result.checks.has(id)) regressions.push(`${label}: ${id} was not reported`);
|
||||
}
|
||||
}
|
||||
for (const [mode, scenarios] of Object.entries(baseline.modes)) {
|
||||
for (const scenario of Object.keys(scenarios)) {
|
||||
if (!results.some((result) => result.mode === mode && result.scenario === scenario)) {
|
||||
regressions.push(`${mode} ${scenario}: scenario did not run`);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (fixed.length > 0) {
|
||||
console.log(
|
||||
`\nFixed since the baseline (run with --update-baseline):\n${fixed.map((line) => ` ${line}`).join("\n")}`,
|
||||
);
|
||||
}
|
||||
if (regressions.length > 0) {
|
||||
console.log(`\nRegressions:\n${regressions.map((line) => ` ${line}`).join("\n")}`);
|
||||
return false;
|
||||
}
|
||||
console.log("\nNo regressions against the baseline.");
|
||||
return true;
|
||||
}
|
||||
|
||||
function printResults(results: ScenarioResult[]): void {
|
||||
for (const mode of MODES) {
|
||||
const modeResults = results.filter((result) => result.mode === mode);
|
||||
if (modeResults.length === 0) continue;
|
||||
console.log(`\n${mode}`);
|
||||
for (const result of modeResults) {
|
||||
const statuses = [...result.checks.values()];
|
||||
const failed = statuses.filter((status) => status === "fail").length;
|
||||
const passed = statuses.length - failed;
|
||||
console.log(` ${failed === 0 ? "pass" : "FAIL"} ${result.scenario} (${passed} passed, ${failed} failed)`);
|
||||
for (const [id, status] of result.checks) {
|
||||
if (status !== "fail") continue;
|
||||
const message = result.messages.get(id)?.replace(/\s+/g, " ").slice(0, 300);
|
||||
console.log(` ${id}${message ? `: ${message}` : ""}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function main(): Promise<number> {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
mode: { type: "string", multiple: true },
|
||||
scenario: { type: "string", multiple: true },
|
||||
"update-baseline": { type: "boolean", default: false },
|
||||
"keep-results": { type: "boolean", default: false },
|
||||
verbose: { type: "boolean", default: false },
|
||||
help: { type: "boolean", short: "h", default: false },
|
||||
},
|
||||
});
|
||||
if (values.help) {
|
||||
console.log(`Usage: node packages/coding-agent/test/mcp-conformance/run.ts [options]
|
||||
|
||||
Options:
|
||||
--mode <version> Only run this protocol version (repeatable): ${MODES.join(", ")}
|
||||
--scenario <name> Only run this scenario (repeatable)
|
||||
--update-baseline Write the results to baseline.json instead of comparing
|
||||
--keep-results Keep the upstream checks.json and client output
|
||||
--verbose Print the upstream runner output`);
|
||||
return 0;
|
||||
}
|
||||
if (process.platform === "win32") {
|
||||
console.error("The conformance runner needs a POSIX shell.");
|
||||
return 1;
|
||||
}
|
||||
const modes = values.mode ?? [...MODES];
|
||||
for (const mode of modes) {
|
||||
if (!(MODES as readonly string[]).includes(mode)) {
|
||||
console.error(`Unsupported mode ${mode}. Supported: ${MODES.join(", ")}`);
|
||||
return 1;
|
||||
}
|
||||
}
|
||||
const partial = values.mode !== undefined || values.scenario !== undefined;
|
||||
if (values["update-baseline"] && partial) {
|
||||
console.error("--update-baseline needs a full run, without --mode or --scenario.");
|
||||
return 1;
|
||||
}
|
||||
|
||||
const workDir = mkdtempSync(join(tmpdir(), "pi-mcp-conformance-"));
|
||||
try {
|
||||
// The upstream runner splits --command on spaces, so use a launcher path without any.
|
||||
const launcher = join(workDir, "pi-client");
|
||||
writeFileSync(
|
||||
launcher,
|
||||
`#!/bin/sh\nexec ${JSON.stringify(process.execPath)} --import ${JSON.stringify(resolverUrl)} ${JSON.stringify(clientPath)} "$@"\n`,
|
||||
);
|
||||
chmodSync(launcher, 0o755);
|
||||
|
||||
const results: ScenarioResult[] = [];
|
||||
for (const mode of modes) {
|
||||
const scenarios = (await listScenarios(mode, workDir)).filter(
|
||||
(scenario) => !values.scenario || values.scenario.includes(scenario),
|
||||
);
|
||||
for (const scenario of scenarios) {
|
||||
process.stderr.write(`${mode} ${scenario}\n`);
|
||||
results.push(await runScenario(mode, scenario, launcher, workDir, values.verbose));
|
||||
}
|
||||
}
|
||||
if (results.length === 0) {
|
||||
console.error("No scenarios matched.");
|
||||
return 1;
|
||||
}
|
||||
printResults(results);
|
||||
|
||||
if (values["update-baseline"]) {
|
||||
writeFileSync(baselinePath, `${JSON.stringify(toBaseline(results), null, "\t")}\n`);
|
||||
console.log(`\nWrote ${baselinePath}`);
|
||||
return 0;
|
||||
}
|
||||
const baseline = loadBaseline();
|
||||
if (!baseline) {
|
||||
console.error("\nNo baseline.json. Run with --update-baseline and review it.");
|
||||
return 1;
|
||||
}
|
||||
if (baseline.conformance !== CONFORMANCE_PACKAGE) {
|
||||
console.error(`\nbaseline.json is for ${baseline.conformance}, not ${CONFORMANCE_PACKAGE}. Regenerate it.`);
|
||||
return 1;
|
||||
}
|
||||
const scoped: Baseline = partial
|
||||
? {
|
||||
conformance: baseline.conformance,
|
||||
modes: Object.fromEntries(
|
||||
modes.map((mode) => [
|
||||
mode,
|
||||
Object.fromEntries(
|
||||
Object.entries(baseline.modes[mode] ?? {}).filter(
|
||||
([scenario]) => !values.scenario || values.scenario.includes(scenario),
|
||||
),
|
||||
),
|
||||
]),
|
||||
),
|
||||
}
|
||||
: baseline;
|
||||
return evaluate(results, scoped) ? 0 : 1;
|
||||
} finally {
|
||||
if (values["keep-results"]) console.log(`\nResults kept in ${workDir}`);
|
||||
else rmSync(workDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
process.exit(await main());
|
||||
Reference in New Issue
Block a user