From 05b72f69879780e3b5c8cfcdd9470f5e5507ae4a Mon Sep 17 00:00:00 2001
From: alexion <contact@alexion.dev>
Date: Fri, 17 Jul 2026 11:55:53 -0400
Subject: [PATCH] fix: ignore markdown emphasis when matching read answers
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The read checker matched a task's required-fact phrasings as plain
substrings of the agent's report after only lowercasing and collapsing
whitespace. An answer that was substantively correct but wrapped a value in
markdown (e.g. `**5**`) failed the match, because the emphasis markers broke
the phrase adjacency (`**5** open` does not contain `5 open`) — a correct
answer scored incorrect on formatting alone.

Strip markdown emphasis/code markers (`*`, `_`, backtick) during
normalization so the match is on substance, not presentation. A guard test
confirms a wrong value still fails after stripping.
---
 bench/checker.test.ts | 30 ++++++++++++++++++++++++++++++
 bench/checker.ts      | 13 +++++++++++--
 2 files changed, 41 insertions(+), 2 deletions(-)

diff --git a/bench/checker.test.ts b/bench/checker.test.ts
index e6c3ed3..ddaebe7 100644
--- a/bench/checker.test.ts
+++ b/bench/checker.test.ts
@@ -310,6 +310,36 @@ describe("checkReadAnswer", () => {
 
     expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
   });
+
+  it("passes when the report wraps a fact's phrasing in markdown emphasis", () => {
+    // The fact's acceptable phrasing is the bare substring "5 open".
+    const facts: RequiredFact[] = [
+      { description: "open-issue count", anyOf: ["5 open"] },
+    ];
+
+    // The report is correct but renders the number in markdown bold. Emphasis
+    // markers (`*`, `_`, backtick) are formatting, not substance, so "**5**" is
+    // equivalent to "5" and the phrasing "5 open" is present.
+    const report = "There are **5** open issues in the repository.";
+
+    expect(checkReadAnswer(facts, report)).toEqual({ pass: true });
+  });
+
+  it("fails when a markdown-formatted report states the wrong value", () => {
+    // The only acceptable phrasing counts five open issues.
+    const facts: RequiredFact[] = [
+      { description: "open-issue count", anyOf: ["5 open"] },
+    ];
+
+    // The report is markdown-formatted but substantively wrong: it counts three,
+    // not five. Stripping emphasis must only remove formatting, so after stripping
+    // ("there are 3 open issues") the phrasing "5 open" is still absent.
+    const report = "There are **3** open issues in the repository.";
+
+    const result = checkReadAnswer(facts, report);
+
+    expect(result.pass).toBe(false);
+  });
 });
 
 describe("score", () => {
diff --git a/bench/checker.ts b/bench/checker.ts
index 082d5e2..c51a217 100644
--- a/bench/checker.ts
+++ b/bench/checker.ts
@@ -253,9 +253,18 @@ export function checkReadAnswer(facts: RequiredFact[], report: string): CheckRes
   };
 }
 
-/** Lower-case and collapse runs of whitespace so incidental phrasing does not matter. */
+/**
+ * Lower-case, drop markdown emphasis/code markers, and collapse runs of
+ * whitespace so incidental phrasing and formatting do not matter — a report that
+ * bolds a value (`**5**`) matches a phrasing that does not (`5 open`), since the
+ * emphasis is presentation, not substance.
+ */
 function normalizeText(text: string): string {
-  return text.toLowerCase().replace(/\s+/g, " ").trim();
+  return text
+    .toLowerCase()
+    .replace(/[*_`]/g, "")
+    .replace(/\s+/g, " ")
+    .trim();
 }
 
 /**