alexion
6081e3e325
Imported as a linear vendor commit so forge rebases do not replay the upstream root tree. git-subtree-dir: third_party/pi git-subtree-split: 71023aec1546d6d677a65669c5665c0927ce0dd0
67 lines
3.0 KiB
TypeScript
67 lines
3.0 KiB
TypeScript
import { globSync } from "node:fs";
|
|
import { resolve } from "node:path";
|
|
import { Type } from "@earendil-works/pi-ai";
|
|
import { defineTool } from "@earendil-works/pi-coding-agent";
|
|
import { expect } from "vitest";
|
|
import { describeEval, toolCalls } from "vitest-evals";
|
|
import { createPiCodingAgentHarness } from "../src/harness.ts";
|
|
|
|
const TOOL_NAME = "submit_documentation_audit";
|
|
const submitAudit = defineTool({
|
|
name: TOOL_NAME,
|
|
label: "Submit documentation audit",
|
|
description: "Submit the final verdict after completing the documentation investigation.",
|
|
promptSnippet: "Submit the final documentation audit as validated structured data",
|
|
parameters: Type.Object(
|
|
{
|
|
verdict: Type.Union([Type.Literal("match"), Type.Literal("mismatch"), Type.Literal("inconclusive")]),
|
|
explanation: Type.String({ minLength: 1, maxLength: 2000 }),
|
|
},
|
|
{ additionalProperties: false },
|
|
),
|
|
constrainedSampling: { type: "json_schema", strict: "prefer" },
|
|
async execute(_toolCallId, params) {
|
|
return {
|
|
content: [{ type: "text", text: "Documentation audit submitted." }],
|
|
details: params,
|
|
terminate: true,
|
|
};
|
|
},
|
|
});
|
|
|
|
const repositoryRoot = resolve(import.meta.dirname, "../../..");
|
|
const docsRoot = resolve(repositoryRoot, "packages/coding-agent/docs");
|
|
const pages = globSync("**/*.md", { cwd: docsRoot })
|
|
.map((path) => ({ path: path.replaceAll("\\", "/") }))
|
|
.sort((left, right) => left.path.localeCompare(right.path));
|
|
const harness = createPiCodingAgentHarness({
|
|
name: "documentation-page-audit",
|
|
tools: ["read", "grep", "find", "ls", TOOL_NAME],
|
|
customTools: [submitAudit],
|
|
});
|
|
|
|
describeEval("Audit documentation against implementation", { harness }, (it) => {
|
|
it.for(pages)("$path matches the implementation", async ({ path }, { run }) => {
|
|
const documentationPath = resolve(docsRoot, path);
|
|
const result = await run(`Audit this Pi documentation page against the repository implementation.
|
|
|
|
Documentation page: ${documentationPath}
|
|
Repository root: ${repositoryRoot}
|
|
|
|
Read the complete page and the relevant implementation. Report a mismatch only for a clear, user-visible contradiction between an explicit documentation claim and actual runtime behavior. Follow the runtime path; names, comments, types, isolated helpers, and tests are not sufficient evidence by themselves.
|
|
|
|
Missing internal detail, ambiguous wording, hypothetical misuse, and undocumented edge cases are not mismatches. If the evidence is not decisive, report inconclusive. Otherwise report match.
|
|
|
|
For a mismatch, quote the claim, cite the implementation path and symbol, and state the concrete behavior a user would observe.
|
|
|
|
Call ${TOOL_NAME} exactly once as your final action. Do not return prose.`);
|
|
|
|
const auditCalls = toolCalls(result).filter((call) => call.name === TOOL_NAME);
|
|
expect(auditCalls).toHaveLength(1);
|
|
const auditCall = auditCalls[0];
|
|
expect(auditCall?.status).toBe("ok");
|
|
const explanation = auditCall?.arguments?.explanation;
|
|
expect(auditCall?.arguments?.verdict, typeof explanation === "string" ? explanation : undefined).toBe("match");
|
|
});
|
|
});
|