fix: ignore markdown emphasis when matching read answers
Some checks failed
CI / test (pull_request) Failing after 51s
Some checks failed
CI / test (pull_request) Failing after 51s
The read checker matched a task's required-fact phrasings as plain substrings of the agent's report after only lowercasing and collapsing whitespace. An answer that was substantively correct but wrapped a value in markdown (e.g. `**5**`) failed the match, because the emphasis markers broke the phrase adjacency (`**5** open` does not contain `5 open`) — a correct answer scored incorrect on formatting alone. Strip markdown emphasis/code markers (`*`, `_`, backtick) during normalization so the match is on substance, not presentation. A guard test confirms a wrong value still fails after stripping.
This commit is contained in:
@@ -310,6 +310,36 @@ describe("checkReadAnswer", () => {
|
|||||||
|
|
||||||
expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
|
expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("passes when the report wraps a fact's phrasing in markdown emphasis", () => {
|
||||||
|
// The fact's acceptable phrasing is the bare substring "5 open".
|
||||||
|
const facts: RequiredFact[] = [
|
||||||
|
{ description: "open-issue count", anyOf: ["5 open"] },
|
||||||
|
];
|
||||||
|
|
||||||
|
// The report is correct but renders the number in markdown bold. Emphasis
|
||||||
|
// markers (`*`, `_`, backtick) are formatting, not substance, so "**5**" is
|
||||||
|
// equivalent to "5" and the phrasing "5 open" is present.
|
||||||
|
const report = "There are **5** open issues in the repository.";
|
||||||
|
|
||||||
|
expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
|
||||||
|
});
|
||||||
|
|
||||||
|
it("fails when a markdown-formatted report states the wrong value", () => {
|
||||||
|
// The only acceptable phrasing counts five open issues.
|
||||||
|
const facts: RequiredFact[] = [
|
||||||
|
{ description: "open-issue count", anyOf: ["5 open"] },
|
||||||
|
];
|
||||||
|
|
||||||
|
// The report is markdown-formatted but substantively wrong: it counts three,
|
||||||
|
// not five. Stripping emphasis must only remove formatting, so after stripping
|
||||||
|
// ("there are 3 open issues") the phrasing "5 open" is still absent.
|
||||||
|
const report = "There are **3** open issues in the repository.";
|
||||||
|
|
||||||
|
const result = checkReadAnswer(facts, report);
|
||||||
|
|
||||||
|
expect(result.pass).toBe(false);
|
||||||
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
describe("score", () => {
|
describe("score", () => {
|
||||||
|
|||||||
@@ -253,9 +253,18 @@ export function checkReadAnswer(facts: RequiredFact[], report: string): CheckRes
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
/** Lower-case and collapse runs of whitespace so incidental phrasing does not matter. */
|
/**
|
||||||
|
* Lower-case, drop markdown emphasis/code markers, and collapse runs of
|
||||||
|
* whitespace so incidental phrasing and formatting do not matter — a report that
|
||||||
|
* bolds a value (`**5**`) matches a phrasing that does not (`5 open`), since the
|
||||||
|
* emphasis is presentation, not substance.
|
||||||
|
*/
|
||||||
function normalizeText(text: string): string {
|
function normalizeText(text: string): string {
|
||||||
return text.toLowerCase().replace(/\s+/g, " ").trim();
|
return text
|
||||||
|
.toLowerCase()
|
||||||
|
.replace(/[*_`]/g, "")
|
||||||
|
.replace(/\s+/g, " ")
|
||||||
|
.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
|
|||||||
Reference in New Issue
Block a user