alexion
e35ed9d05b
Introduce the thinnest complete path that benchmarks a skill's efficacy against a no-skill baseline and renders an HTML report. - tests/ convention: a top-level tree mirroring skills/ by full path, with case directories holding a case.md (CASE-FORMAT.md) and an optional fixture. - benchmark-skill: a distributed, non-model-invocable runner (/benchmark-skill <name>) that orchestrates force-invoked new-skill and skill-absent baseline arms plus a blind per-trial judge as in-session subagents. - Deterministic core (Python): a pure transform over the collected run data that sums per-arm usage, collapses each trial to WIN/TIE/LOSS, applies the efficacy pass rule (wins >= 3, losses <= 1), and renders a self-contained report. Parameterized over arms and comparisons; two-arm here. - Fixture-driven no-LLM check (checks/benchmark-core.nix) wired into flake.nix. - Real tests tree for axi-review; a live run produced its efficacy report.
82 lines
3.5 KiB
Nix
82 lines
3.5 KiB
Nix
# Feeds the committed run-bundle fixture through the benchmark skill's
|
|
# deterministic core and asserts the per-arm metrics, the per-case and
|
|
# skill-level Efficacy verdict, and the rendered report.
|
|
# No LLM runs.
|
|
{
|
|
pkgs,
|
|
}:
|
|
let
|
|
core = ../skills/benchmark-skill/core/benchmark_core.py;
|
|
fixture = ./fixtures/benchmark/run-bundle.json;
|
|
in
|
|
pkgs.runCommandLocal "benchmark-core-check"
|
|
{
|
|
nativeBuildInputs = [ pkgs.python3 ];
|
|
inherit core fixture;
|
|
}
|
|
''
|
|
fail() { echo "FAIL: $1" >&2; exit 1; }
|
|
|
|
echo "the core turns a run bundle into a results model and an HTML report"
|
|
python3 "$core" "$fixture" --json results.json --html report.html \
|
|
|| fail "the core exited non-zero"
|
|
|
|
echo "the results model carries the expected metrics and verdicts"
|
|
python3 - results.json <<'PY' || fail "a results-model assertion failed"
|
|
import json, sys
|
|
r = json.load(open(sys.argv[1]))
|
|
|
|
assert r["armShape"] == "two-arm", r["armShape"]
|
|
# The skill verdict is red because two of the three cases fail efficacy.
|
|
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
|
|
|
|
new = r["armMetrics"]["new"]
|
|
assert new["turns"] == 45, new["turns"]
|
|
assert new["rawTokens"] == 171000, new["rawTokens"]
|
|
assert new["costEquivalentTokens"] == 184500, new["costEquivalentTokens"]
|
|
assert abs(new["imputedCost"] - 0.9225) < 1e-9, new["imputedCost"]
|
|
|
|
base = r["armMetrics"]["baseline"]
|
|
assert base["turns"] == 30, base["turns"]
|
|
assert base["rawTokens"] == 85500, base["rawTokens"]
|
|
assert base["costEquivalentTokens"] == 92250, base["costEquivalentTokens"]
|
|
|
|
cases = {c["name"]: c for c in r["cases"]}
|
|
# Authored order is preserved, never reshuffled by verdict.
|
|
assert [c["name"] for c in r["cases"]] == ["clean-pass", "regresses-baseline", "hard-gate-fail"]
|
|
|
|
a = cases["clean-pass"]
|
|
assert a["efficacyPassed"] is True
|
|
assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"]
|
|
assert a["comparisons"]["efficacy"]["wins"] == 4
|
|
assert a["comparisons"]["efficacy"]["losses"] == 0
|
|
|
|
# A head-to-head fail: two losses drop it under the wins>=3, losses<=1 rule,
|
|
# and both losses are flagged for human review.
|
|
b = cases["regresses-baseline"]
|
|
assert b["efficacyPassed"] is False
|
|
assert b["comparisons"]["efficacy"]["losses"] == 2
|
|
assert b["comparisons"]["efficacy"]["flaggedLosses"] == [3, 4]
|
|
|
|
# A hard-assertion failure fails the case outright despite a clean sweep.
|
|
c = cases["hard-gate-fail"]
|
|
assert c["hardFailed"] is True
|
|
assert c["efficacyPassed"] is False
|
|
assert c["comparisons"]["efficacy"]["wins"] == 5
|
|
print("results-model assertions passed")
|
|
PY
|
|
|
|
echo "the report is self-contained and shows the badge, cost table, and cases"
|
|
grep -q '<!doctype html>' report.html || fail "report is not a self-contained document"
|
|
grep -q 'Efficacy: RED' report.html || fail "report is missing the red Efficacy badge"
|
|
grep -q 'Cost-equiv tokens' report.html || fail "report is missing the per-arm cost table"
|
|
grep -q 'shared-context cache' report.html || fail "report is missing the cache-overhead footnote"
|
|
for name in clean-pass regresses-baseline hard-gate-fail; do
|
|
grep -q "$name" report.html || fail "report omits case $name"
|
|
done
|
|
grep -q 'class="cell LOSS"' report.html || fail "report is missing a per-trial LOSS cell"
|
|
grep -q 'Hard assertion failed' report.html || fail "report does not flag the hard-assertion failure"
|
|
|
|
touch "$out"
|
|
''
|