feat: benchmark credential parity, transcript, honest results + read-tier accuracy (tasks 0032, 0033) #36

Merged
alexion merged 10 commits from task-0032-bench-read-report-persistence into main 2026-07-18 10:28:33 -04:00
2 changed files with 41 additions and 2 deletions
Showing only changes of commit 05b72f6987 - Show all commits

View File

@@ -310,6 +310,36 @@ describe("checkReadAnswer", () => {
expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
});
it("passes when the report wraps a fact's phrasing in markdown emphasis", () => {
// The fact's acceptable phrasing is the bare substring "5 open".
const facts: RequiredFact[] = [
{ description: "open-issue count", anyOf: ["5 open"] },
];
// The report is correct but renders the number in markdown bold. Emphasis
// markers (`*`, `_`, backtick) are formatting, not substance, so "**5**" is
// equivalent to "5" and the phrasing "5 open" is present.
const report = "There are **5** open issues in the repository.";
expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
});
it("fails when a markdown-formatted report states the wrong value", () => {
// The only acceptable phrasing counts five open issues.
const facts: RequiredFact[] = [
{ description: "open-issue count", anyOf: ["5 open"] },
];
// The report is markdown-formatted but substantively wrong: it counts three,
// not five. Stripping emphasis must only remove formatting, so after stripping
// ("there are 3 open issues") the phrasing "5 open" is still absent.
const report = "There are **3** open issues in the repository.";
const result = checkReadAnswer(facts, report);
expect(result.pass).toBe(false);
});
});
describe("score", () => {

View File

@@ -253,9 +253,18 @@ export function checkReadAnswer(facts: RequiredFact[], report: string): CheckRes
};
}
/** Lower-case and collapse runs of whitespace so incidental phrasing does not matter. */
/**
* Lower-case, drop markdown emphasis/code markers, and collapse runs of
* whitespace so incidental phrasing and formatting do not matter — a report that
* bolds a value (`**5**`) matches a phrasing that does not (`5 open`), since the
* emphasis is presentation, not substance.
*/
function normalizeText(text: string): string {
return text.toLowerCase().replace(/\s+/g, " ").trim();
return text
.toLowerCase()
.replace(/[*_`]/g, "")
.replace(/\s+/g, " ")
.trim();
}
/**