# Feeds the committed run-bundle fixture through the benchmark skill's # deterministic core and asserts the per-arm metrics, the per-case and # skill-level Efficacy verdict, and the rendered report. # No LLM runs. { pkgs, }: let core = ../skills/benchmark-skill/core/benchmark_core.py; fixture = ./fixtures/benchmark/run-bundle.json; in pkgs.runCommandLocal "benchmark-core-check" { nativeBuildInputs = [ pkgs.python3 ]; inherit core fixture; } '' fail() { echo "FAIL: $1" >&2; exit 1; } echo "the core turns a run bundle into a results model and an HTML report" python3 "$core" "$fixture" --json results.json --html report.html \ || fail "the core exited non-zero" echo "the results model carries the expected metrics and verdicts" python3 - results.json <<'PY' || fail "a results-model assertion failed" import json, sys r = json.load(open(sys.argv[1])) assert r["armShape"] == "two-arm", r["armShape"] # The skill verdict is red because two of the three cases fail efficacy. assert r["efficacyVerdict"] == "red", r["efficacyVerdict"] new = r["armMetrics"]["new"] assert new["turns"] == 45, new["turns"] assert new["rawTokens"] == 171000, new["rawTokens"] assert new["costEquivalentTokens"] == 184500, new["costEquivalentTokens"] assert abs(new["imputedCost"] - 0.9225) < 1e-9, new["imputedCost"] base = r["armMetrics"]["baseline"] assert base["turns"] == 30, base["turns"] assert base["rawTokens"] == 85500, base["rawTokens"] assert base["costEquivalentTokens"] == 92250, base["costEquivalentTokens"] cases = {c["name"]: c for c in r["cases"]} # Authored order is preserved, never reshuffled by verdict. assert [c["name"] for c in r["cases"]] == ["clean-pass", "regresses-baseline", "hard-gate-fail"] a = cases["clean-pass"] assert a["efficacyPassed"] is True assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"] assert a["comparisons"]["efficacy"]["wins"] == 4 assert a["comparisons"]["efficacy"]["losses"] == 0 # A head-to-head fail: two losses drop it under the wins>=3, losses<=1 rule, # and both losses are flagged for human review. b = cases["regresses-baseline"] assert b["efficacyPassed"] is False assert b["comparisons"]["efficacy"]["losses"] == 2 assert b["comparisons"]["efficacy"]["flaggedLosses"] == [3, 4] # A hard-assertion failure fails the case outright despite a clean sweep. c = cases["hard-gate-fail"] assert c["hardFailed"] is True assert c["efficacyPassed"] is False assert c["comparisons"]["efficacy"]["wins"] == 5 print("results-model assertions passed") PY echo "the report is self-contained and shows the badge, cost table, and cases" grep -q '' report.html || fail "report is not a self-contained document" grep -q 'Efficacy: RED' report.html || fail "report is missing the red Efficacy badge" grep -q 'Cost-equiv tokens' report.html || fail "report is missing the per-arm cost table" grep -q 'shared-context cache' report.html || fail "report is missing the cache-overhead footnote" for name in clean-pass regresses-baseline hard-gate-fail; do grep -q "$name" report.html || fail "report omits case $name" done grep -q 'class="cell LOSS"' report.html || fail "report is missing a per-trial LOSS cell" grep -q 'Hard assertion failed' report.html || fail "report does not flag the hard-assertion failure" touch "$out" ''