alexion/skills
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove benchmark skill

Browse Source
This commit is contained in:
alexion
2026-07-31 18:50:42 -04:00
parent 8479ad096c
commit d3bc9a5760
11 changed files with 4 additions and 4184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@@ -6,12 +6,8 @@ result-*
# Ignore automatically generated direnv output # Ignore automatically generated direnv output
.direnv .direnv
# Benchmark run artifacts: reports and local history, keyed by skill name.
tests/.reports/
# BEGIN mkSkillsShellHook # BEGIN mkSkillsShellHook
# Generated by mkSkillsShellHook. Nix-delivered skill symlinks, kept out of git. # Generated by mkSkillsShellHook. Nix-delivered skill symlinks, kept out of git.
.claude/skills .claude/skills
.agents/skills/gitea-axi .agents/skills/gitea-axi
.agents/skills/benchmark-skill
# END mkSkillsShellHook # END mkSkillsShellHook

4
AGENTS.md
View File

@@ -10,14 +10,12 @@ The domain glossary lives at `$(xdg-user-dir DOCUMENTS)/ai-artifacts/projects/sk
- Skill source lives under `skills/<name>/`. - Skill source lives under `skills/<name>/`.
Do not edit generated links under `.agents/skills/` or `.claude/skills/` as source. Do not edit generated links under `.agents/skills/` or `.claude/skills/` as source.
- Run `nix flake check` before considering a repository-wide change complete. - Run `nix flake check` before considering a repository-wide change complete.
The check builds every packaged skill plus the Home Manager module, shell hook, and benchmark core tests. The check builds every packaged skill plus the Home Manager module and shell hook.
- The dev shell may place Nix-delivered skills into `.agents/skills/` and maintain `.claude/skills` as a compatibility symlink. - The dev shell may place Nix-delivered skills into `.agents/skills/` and maintain `.claude/skills` as a compatibility symlink.
Generated ignore entries belong in the root `.gitignore`, not inside `.agents/skills/`, because skill scanners honor ignore files inside scanned skill directories. Generated ignore entries belong in the root `.gitignore`, not inside `.agents/skills/`, because skill scanners honor ignore files inside scanned skill directories.
## Gotchas ## Gotchas
- `python3` is supplied by the default Nix dev shell, not the ambient environment.
Enter `nix develop` before running Python-based project tools.
- A skill-local `GLOSSARY.md` is runtime reference for that skill and has no relationship to the project's AI-artifacts-vault context glossary. - A skill-local `GLOSSARY.md` is runtime reference for that skill and has no relationship to the project's AI-artifacts-vault context glossary.
- This repository owns packaged skill sources only. - This repository owns packaged skill sources only.
Enabling, replacing, or removing them in a consumer repository such as dotfiles is outside its scope. Enabling, replacing, or removing them in a consumer repository such as dotfiles is outside its scope.

358
checks/benchmark-core.nix
View File

@@ -1,358 +0,0 @@
# Feeds the committed run-bundle fixtures through the benchmark skill's
# deterministic core and asserts the per-arm metrics, the per-case and
# skill-level Efficacy and Regression verdicts, and the rendered report — for
# both the three-arm shape and the degenerate two-arm "no previous version"
# shape.
# It also drives the longitudinal layer: the per-skill history append-and-trim,
# the two trend ribbons including the two-arm gap in the regression series, and
# the per-badge fragility chips on a clean run and a green-but-fragile one.
# Last it drives the all-skills leaderboard: the tiered sort, the not-applicable
# Regression cell, and the links out to each per-skill report.
# No LLM runs.
{
pkgs,
}:
let
core = ../skills/benchmark-skill/core/benchmark_core.py;
threeArm = ./fixtures/benchmark/three-arm-bundle.json;
twoArm = ./fixtures/benchmark/two-arm-bundle.json;
clean = ./fixtures/benchmark/clean-bundle.json;
fragile = ./fixtures/benchmark/fragile-bundle.json;
in
pkgs.runCommandLocal "benchmark-core-check"
{
nativeBuildInputs = [ pkgs.python3 ];
inherit core threeArm twoArm clean fragile;
}
''
fail() { echo "FAIL: $1" >&2; exit 1; }
# --- three-arm run --------------------------------------------------------
echo "the core scores a three-arm run and renders its report"
python3 "$core" "$threeArm" --json three.json --html three.html \
|| fail "the core exited non-zero on the three-arm bundle"
echo "the three-arm results model carries both verdicts, net margins, and metrics"
python3 - three.json <<'PY' || fail "a three-arm results-model assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["armShape"] == "three-arm", r["armShape"]
# Efficacy is red: dead-weight fails the wins floor and hard-gate-fail trips
# the gate. Regression is red: regressed-still-valuable loses twice to the
# previous version.
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
assert r["regressionVerdict"] == "red", r["regressionVerdict"]
assert r["efficacyNetMargin"] == 12, r["efficacyNetMargin"]
assert r["regressionNetMargin"] == 3, r["regressionNetMargin"]
new = r["armMetrics"]["new"]
assert new["turns"] == 60, new["turns"]
assert new["rawTokens"] == 228000, new["rawTokens"]
assert new["costEquivalentTokens"] == 246000, new["costEquivalentTokens"]
assert abs(new["imputedCost"] - 1.23) < 1e-9, new["imputedCost"]
base = r["armMetrics"]["baseline"]
assert base["turns"] == 40, base["turns"]
assert base["rawTokens"] == 114000, base["rawTokens"]
assert base["costEquivalentTokens"] == 123000, base["costEquivalentTokens"]
prev = r["armMetrics"]["previous"]
assert prev["turns"] == 60, prev["turns"]
assert prev["rawTokens"] == 174000, prev["rawTokens"]
assert prev["costEquivalentTokens"] == 195500, prev["costEquivalentTokens"]
assert abs(prev["imputedCost"] - 0.9775) < 1e-9, prev["imputedCost"]
# Cost-table rows render in arm order: new-skill, previous-version, no-skill.
assert [a["id"] for a in r["arms"]] == ["new", "previous", "baseline"], \
[a["id"] for a in r["arms"]]
cases = {c["name"]: c for c in r["cases"]}
# Authored order is preserved, never reshuffled by verdict.
assert [c["name"] for c in r["cases"]] == [
"clean-both", "regressed-still-valuable", "dead-weight", "hard-gate-fail"
]
a = cases["clean-both"]
assert a["efficacyPassed"] is True
assert a["regressionPassed"] is True
assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"]
assert a["comparisons"]["regression"]["trials"] == ["TIE", "TIE", "WIN", "TIE", "TIE"]
# Efficacy green but regression red — the exact crossing this feature exists
# to catch: still beats no-skill, yet degraded from the released version.
b = cases["regressed-still-valuable"]
assert b["efficacyPassed"] is True
assert b["regressionPassed"] is False
assert b["comparisons"]["regression"]["losses"] == 2
assert b["comparisons"]["regression"]["flaggedLosses"] == [0, 1]
# Efficacy red but regression green — already dead weight, but the edit did
# not make it worse.
c = cases["dead-weight"]
assert c["efficacyPassed"] is False
assert c["regressionPassed"] is True
assert c["comparisons"]["efficacy"]["wins"] == 0
assert c["comparisons"]["efficacy"]["losses"] == 1
assert c["comparisons"]["efficacy"]["flaggedLosses"] == [2]
assert c["comparisons"]["regression"]["wins"] == 3
# The hard gate fails efficacy outright but does not gate regression, which
# is judged purely on losses.
d = cases["hard-gate-fail"]
assert d["hardFailed"] is True
assert d["efficacyPassed"] is False
assert d["regressionPassed"] is True
assert d["comparisons"]["efficacy"]["wins"] == 5
print("three-arm results-model assertions passed")
PY
echo "the three-arm report shows both badges, the three-row table, and evidence"
grep -q '<!doctype html>' three.html || fail "three-arm report is not self-contained"
grep -q 'Efficacy: RED' three.html || fail "three-arm report is missing the red Efficacy badge"
grep -q 'Regression: RED' three.html || fail "three-arm report is missing the red Regression badge"
grep -qi 'regress' three.html || fail "three-arm report is missing the verdict headline"
grep -q 'Cost-equiv tokens' three.html || fail "three-arm report is missing the cost table"
grep -q 'Previous version' three.html || fail "three-arm report is missing the previous-version row"
grep -q 'shared-context cache' three.html || fail "three-arm report is missing the cache footnote"
grep -q 'new-vs-no-skill' three.html || fail "three-arm footnote omits the new-vs-no-skill ratio"
grep -q 'new-vs-old' three.html || fail "three-arm footnote omits the new-vs-old ratio"
for name in clean-both regressed-still-valuable dead-weight hard-gate-fail; do
grep -q "$name" three.html || fail "three-arm report omits case $name"
done
grep -q 'class="cell LOSS"' three.html || fail "three-arm report is missing a LOSS cell"
grep -q 'Hard assertion failed' three.html || fail "three-arm report does not flag the hard failure"
grep -q ' open>' three.html || fail "three-arm report does not auto-expand a failing case"
grep -q 'class="cmps"' three.html || fail "three-arm report does not lay comparisons side by side"
# Evidence for the failed regression comparison: new vs previous on trial 1.
grep -q 'OUT-new-regressed-still-valuable-t0' three.html || fail "missing new evidence for the regression loss"
grep -q 'OUT-previous-regressed-still-valuable-t0' three.html || fail "missing previous evidence for the regression loss"
# Evidence for the failed efficacy comparison: new vs baseline on trial 3.
grep -q 'OUT-new-dead-weight-t2' three.html || fail "missing new evidence for the efficacy loss"
grep -q 'OUT-baseline-dead-weight-t2' three.html || fail "missing baseline evidence for the efficacy loss"
# A clean, collapsed case emits no losing-trial evidence.
if grep -q 'OUT-new-clean-both' three.html; then fail "a clean case leaked losing-trial evidence"; fi
# --- two-arm run ----------------------------------------------------------
echo "the core scores the degenerate two-arm run"
python3 "$core" "$twoArm" --json two.json --html two.html \
|| fail "the core exited non-zero on the two-arm bundle"
echo "the two-arm results model reads regression as not-applicable"
python3 - two.json <<'PY' || fail "a two-arm results-model assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["armShape"] == "two-arm", r["armShape"]
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
assert r["regressionVerdict"] == "not-applicable", r["regressionVerdict"]
assert r["regressionNetMargin"] is None, r["regressionNetMargin"]
assert r["efficacyNetMargin"] == 9, r["efficacyNetMargin"]
new = r["armMetrics"]["new"]
assert new["turns"] == 45, new["turns"]
assert new["rawTokens"] == 171000, new["rawTokens"]
assert new["costEquivalentTokens"] == 184500, new["costEquivalentTokens"]
assert abs(new["imputedCost"] - 0.9225) < 1e-9, new["imputedCost"]
base = r["armMetrics"]["baseline"]
assert base["turns"] == 30, base["turns"]
assert base["rawTokens"] == 85500, base["rawTokens"]
assert base["costEquivalentTokens"] == 92250, base["costEquivalentTokens"]
assert "previous" not in r["armMetrics"], "two-arm run has no previous arm"
assert [c["name"] for c in r["cases"]] == [
"clean-pass", "regresses-baseline", "hard-gate-fail"
]
for c in r["cases"]:
assert c["regressionPassed"] is None, (c["name"], c["regressionPassed"])
b = {c["name"]: c for c in r["cases"]}["regresses-baseline"]
assert b["efficacyPassed"] is False
assert b["comparisons"]["efficacy"]["losses"] == 2
assert b["comparisons"]["efficacy"]["flaggedLosses"] == [3, 4]
h = {c["name"]: c for c in r["cases"]}["hard-gate-fail"]
assert h["hardFailed"] is True
print("two-arm results-model assertions passed")
PY
echo "the two-arm report reads Regression as not-applicable and drops the previous row"
grep -q 'Efficacy: RED' two.html || fail "two-arm report is missing the Efficacy badge"
grep -q 'Regression: N/A' two.html || fail "two-arm report does not read Regression as not-applicable"
grep -q 'new-vs-no-skill' two.html || fail "two-arm footnote omits the new-vs-no-skill ratio"
if grep -q 'new-vs-old' two.html; then fail "two-arm footnote names a new-vs-old ratio that does not apply"; fi
if grep -q 'Previous version' two.html; then fail "two-arm report shows a previous-version row"; fi
grep -q 'OUT-new-regresses-baseline-t3' two.html || fail "missing new evidence for the two-arm efficacy loss"
grep -q 'OUT-baseline-regresses-baseline-t3' two.html || fail "missing baseline evidence for the two-arm efficacy loss"
# --- clean run: fully collapsed, one narrowest-margin chip, ribbons ---------
echo "the core rests a clean run fully collapsed with a lone narrowest-margin chip"
python3 "$core" "$clean" --json clean.json --html clean.html --history clean.history.jsonl \
|| fail "the core exited non-zero on the clean bundle"
python3 - clean.json <<'PY' || fail "a clean-run assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["efficacyVerdict"] == "green", r["efficacyVerdict"]
assert r["regressionVerdict"] == "green", r["regressionVerdict"]
cases = {c["name"]: c for c in r["cases"]}
# The narrowest efficacy case (fewest wins) carries the chip, the roomy one does not.
assert [ch["axis"] for ch in cases["narrowest"]["chips"]] == ["efficacy"], cases["narrowest"]["chips"]
assert cases["roomy"]["chips"] == [], cases["roomy"]["chips"]
# The efficacy chip carries the case's win count, not merely its label text.
assert cases["narrowest"]["chips"][0]["wins"] == 3, cases["narrowest"]["chips"]
print("clean-run assertions passed")
PY
if grep -q ' open>' clean.html; then fail "a clean run did not rest fully collapsed"; fi
grep -q 'class="chip efficacy"' clean.html || fail "clean run omits the narrowest-margin efficacy chip"
if grep -q 'class="chip regression"' clean.html; then fail "clean run shows a spurious regression chip"; fi
grep -q 'class="ribbons"' clean.html || fail "clean run omits the trend ribbons"
grep -q 'class="spark"' clean.html || fail "clean run omits a sparkline"
grep -q 'class="ring"' clean.html || fail "clean run does not ring the current run"
# --- history append-and-trim and the two-ribbon window with a two-arm gap ---
echo "the core appends to and trims the per-skill history and plots the ribbon window"
python3 - seed.history.jsonl <<'PY' || fail "seeding the history fixture failed"
import json, sys
# 55 prior runs, all three-arm except the most recent, which is two-arm and so
# must plot a gap on the regression axis once it lands inside the 7-run window.
lines = []
for i in range(55):
two = i == 54
lines.append({
"generatedAt": f"seed-{i}",
"armShape": "two-arm" if two else "three-arm",
"efficacyNet": i % 5,
"efficacyPass": True,
"regressionNet": None if two else i % 3,
"regressionPass": None if two else True,
})
open(sys.argv[1], "w").write("".join(json.dumps(l) + "\n" for l in lines))
PY
python3 "$core" "$clean" --json trend.json --history seed.history.jsonl \
|| fail "the core exited non-zero on the seeded-history run"
python3 - seed.history.jsonl trend.json <<'PY' || fail "a history/trend assertion failed"
import json, sys
lines = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
# 55 prior + this run = 56, trimmed oldest-first back to the cap of 50.
assert len(lines) == 50, len(lines)
gens = [l["generatedAt"] for l in lines]
assert "seed-0" not in gens and "seed-5" not in gens, "oldest runs were not trimmed"
assert "seed-6" in gens and "seed-54" in gens, "recent runs were wrongly trimmed"
assert lines[-1]["armShape"] == "three-arm", lines[-1]
assert lines[-1]["efficacyNet"] == 8 and lines[-1]["regressionNet"] == 1, lines[-1]
t = json.load(open(sys.argv[2]))["trend"]
eff, reg = t["efficacy"]["points"], t["regression"]["points"]
assert len(eff) == 7 and len(reg) == 7, (len(eff), len(reg))
# Efficacy has a value every run, while regression breaks at the two-arm run.
assert all(p["net"] is not None for p in eff), "efficacy series should have no gaps"
assert eff[-1]["current"] and reg[-1]["current"], "the current run is ringed on both axes"
assert reg[5]["net"] is None, [p["net"] for p in reg]
assert reg[-1]["net"] is not None, "the current three-arm run has a regression dot"
assert t["regression"]["applicable"] == 6, t["regression"]["applicable"]
print("history/trend assertions passed")
PY
# --- green-but-fragile run: independent per-badge chips, both on one case ----
echo "the core chips each fragility axis independently, and both on a doubly-fragile case"
python3 "$core" "$fragile" --json fragile.json --html fragile.html \
|| fail "the core exited non-zero on the fragile bundle"
python3 - fragile.json <<'PY' || fail "a fragility-chip assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["efficacyVerdict"] == "green" and r["regressionVerdict"] == "green"
cases = {c["name"]: c for c in r["cases"]}
# One case is both the narrowest efficacy margin and at exactly one regression
# loss, so it carries both chips.
# The others carry at most their own axis.
assert {ch["axis"] for ch in cases["both"]["chips"]} == {"efficacy", "regression"}, cases["both"]["chips"]
assert [ch["axis"] for ch in cases["reg-only"]["chips"]] == ["regression"], cases["reg-only"]["chips"]
assert cases["none"]["chips"] == [], cases["none"]["chips"]
print("fragility-chip assertions passed")
PY
grep -q 'class="chip efficacy"' fragile.html || fail "fragile run omits the efficacy chip"
grep -q 'class="chip regression"' fragile.html || fail "fragile run omits the regression chip"
# --- all-skills leaderboard: tiered sort, the N/A cell, and per-skill links ---
echo "the core ranks per-skill results into a tiered index leaderboard"
# A synthetic clean-green skill whose narrowest efficacy margin sits above the
# floor, so it is the one input that reaches the clean-green tier. The four
# scored models above supply the other tiers: three-arm regresses (tier 0),
# two-arm fails efficacy with regression not-applicable (tier 1), and the
# clean and fragile green runs are both one flip from failing (tier 2).
python3 - robust.results.json leaky.results.json <<'PY' || fail "writing the leaderboard fixtures failed"
import json, sys
# A clean-green skill whose narrowest efficacy margin clears the floor, the
# one input that reaches the clean-green tier.
json.dump({
"skill": "robust-skill",
"efficacyVerdict": "green",
"regressionVerdict": "green",
"cases": [{"chips": [{"axis": "efficacy", "wins": 5}]}],
}, open(sys.argv[1], "w"))
# An efficacy-red skill whose still-green regression axis sits one loss from
# regressing, so it carries a regression chip yet belongs in the red tier.
json.dump({
"skill": "leaky-skill",
"efficacyVerdict": "red",
"regressionVerdict": "green",
"cases": [{"chips": [{"axis": "regression"}]}],
}, open(sys.argv[2], "w"))
PY
python3 "$core" --leaderboard \
three.json two.json clean.json fragile.json robust.results.json leaky.results.json \
--json board.json --html board.html \
|| fail "the core exited non-zero rendering the leaderboard"
python3 - board.json <<'PY' || fail "a leaderboard assertion failed"
import json, sys
rows = json.load(open(sys.argv[1]))
# Any red first — regressions above efficacy failures, efficacy failures
# ordered by name — then fragile-passing by name, then clean green.
assert [r["skill"] for r in rows] == [
"sample-skill", "leaky-skill", "sample-skill-2arm",
"clean-skill", "fragile-skill", "robust-skill",
], [r["skill"] for r in rows]
assert [r["tier"] for r in rows] == [0, 1, 1, 2, 2, 3], [r["tier"] for r in rows]
by = {r["skill"]: r for r in rows}
# The regressed skill outranks the merely-efficacy-failed ones.
assert by["sample-skill"]["regressionVerdict"] == "red"
# A skill with no previous version reads not-applicable in its Regression cell.
assert by["sample-skill-2arm"]["regressionVerdict"] == "not-applicable"
assert by["sample-skill-2arm"]["efficacyVerdict"] == "red"
# The fragile tier reuses the per-badge chips: a floor-bound efficacy margin,
# a case one loss from regressing, or both.
assert by["clean-skill"]["fragileAxes"] == ["efficacy"], by["clean-skill"]
assert by["fragile-skill"]["fragileAxes"] == ["efficacy", "regression"], by["fragile-skill"]
assert by["robust-skill"]["fragileAxes"] == [], by["robust-skill"]
# A red skill carries no fragility chip even when a still-passing axis is at
# its edge, so the chip stays the marker of the fragile-but-passing tier.
assert by["leaky-skill"]["fragileAxes"] == [], by["leaky-skill"]
# Every row links out to its per-skill report, keyed by skill name.
for r in rows:
assert r["href"] == r["skill"] + ".html", r
print("leaderboard assertions passed")
PY
echo "the leaderboard is self-contained and links out to each per-skill report"
grep -q '<!doctype html>' board.html || fail "the leaderboard is not self-contained"
for skill in sample-skill sample-skill-2arm clean-skill fragile-skill robust-skill leaky-skill; do
grep -q "href=\"$skill.html\"" board.html || fail "the leaderboard omits the link to $skill"
done
grep -q '>N/A<' board.html || fail "the leaderboard omits the not-applicable Regression cell"
grep -q 'near regressing' board.html || fail "the leaderboard omits the regression fragility chip"
grep -q 'narrow efficacy' board.html || fail "the leaderboard omits the efficacy fragility chip"
touch "$out"
''

496
checks/fixtures/benchmark/clean-bundle.json
View File

@@ -1,496 +0,0 @@
{
"skill": "clean-skill",
"generatedAt": "2026-07-24T10:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "narrowest",
"description": "narrowest scenario",
"softCriteria": [
"a good answer for narrowest"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t0",
"previous": "OUT-previous-narrowest-t0",
"baseline": "OUT-baseline-narrowest-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t1",
"previous": "OUT-previous-narrowest-t1",
"baseline": "OUT-baseline-narrowest-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t2",
"previous": "OUT-previous-narrowest-t2",
"baseline": "OUT-baseline-narrowest-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t3",
"previous": "OUT-previous-narrowest-t3",
"baseline": "OUT-baseline-narrowest-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t4",
"previous": "OUT-previous-narrowest-t4",
"baseline": "OUT-baseline-narrowest-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "roomy",
"description": "roomy scenario",
"softCriteria": [
"a good answer for roomy"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "new",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t0",
"previous": "OUT-previous-roomy-t0",
"baseline": "OUT-baseline-roomy-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t1",
"previous": "OUT-previous-roomy-t1",
"baseline": "OUT-baseline-roomy-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t2",
"previous": "OUT-previous-roomy-t2",
"baseline": "OUT-baseline-roomy-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t3",
"previous": "OUT-previous-roomy-t3",
"baseline": "OUT-baseline-roomy-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t4",
"previous": "OUT-previous-roomy-t4",
"baseline": "OUT-baseline-roomy-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
}
]
}

725
checks/fixtures/benchmark/fragile-bundle.json
View File

@@ -1,725 +0,0 @@
{
"skill": "fragile-skill",
"generatedAt": "2026-07-24T11:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "both",
"description": "both scenario",
"softCriteria": [
"a good answer for both"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "new",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t0",
"previous": "OUT-previous-both-t0",
"baseline": "OUT-baseline-both-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t1",
"previous": "OUT-previous-both-t1",
"baseline": "OUT-baseline-both-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t2",
"previous": "OUT-previous-both-t2",
"baseline": "OUT-baseline-both-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t3",
"previous": "OUT-previous-both-t3",
"baseline": "OUT-baseline-both-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "previous",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t4",
"previous": "OUT-previous-both-t4",
"baseline": "OUT-baseline-both-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "reg-only",
"description": "reg-only scenario",
"softCriteria": [
"a good answer for reg-only"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t0",
"previous": "OUT-previous-reg-only-t0",
"baseline": "OUT-baseline-reg-only-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t1",
"previous": "OUT-previous-reg-only-t1",
"baseline": "OUT-baseline-reg-only-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t2",
"previous": "OUT-previous-reg-only-t2",
"baseline": "OUT-baseline-reg-only-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t3",
"previous": "OUT-previous-reg-only-t3",
"baseline": "OUT-baseline-reg-only-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "previous",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t4",
"previous": "OUT-previous-reg-only-t4",
"baseline": "OUT-baseline-reg-only-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "none",
"description": "none scenario",
"softCriteria": [
"a good answer for none"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t0",
"previous": "OUT-previous-none-t0",
"baseline": "OUT-baseline-none-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t1",
"previous": "OUT-previous-none-t1",
"baseline": "OUT-baseline-none-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t2",
"previous": "OUT-previous-none-t2",
"baseline": "OUT-baseline-none-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t3",
"previous": "OUT-previous-none-t3",
"baseline": "OUT-baseline-none-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t4",
"previous": "OUT-previous-none-t4",
"baseline": "OUT-baseline-none-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
}
]
}

954
checks/fixtures/benchmark/three-arm-bundle.json
View File

@@ -1,954 +0,0 @@
{
"skill": "sample-skill",
"generatedAt": "2026-07-24T12:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "clean-both",
"description": "New beats baseline and holds steady versus the previous version.",
"softCriteria": [
"The answer is grounded in the fixture."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t0",
"previous": "OUT-previous-clean-both-t0",
"baseline": "OUT-baseline-clean-both-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t1",
"previous": "OUT-previous-clean-both-t1",
"baseline": "OUT-baseline-clean-both-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-clean-both-t2",
"previous": "OUT-previous-clean-both-t2",
"baseline": "OUT-baseline-clean-both-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t3",
"previous": "OUT-previous-clean-both-t3",
"baseline": "OUT-baseline-clean-both-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t4",
"previous": "OUT-previous-clean-both-t4",
"baseline": "OUT-baseline-clean-both-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "regressed-still-valuable",
"description": "New still beats baseline but has degraded from the previous version.",
"softCriteria": [
"The answer resolves the user's request."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "previous",
"rationale": "judge: regression previous"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t0",
"previous": "OUT-previous-regressed-still-valuable-t0",
"baseline": "OUT-baseline-regressed-still-valuable-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "previous",
"rationale": "judge: regression previous"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t1",
"previous": "OUT-previous-regressed-still-valuable-t1",
"baseline": "OUT-baseline-regressed-still-valuable-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t2",
"previous": "OUT-previous-regressed-still-valuable-t2",
"baseline": "OUT-baseline-regressed-still-valuable-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t3",
"previous": "OUT-previous-regressed-still-valuable-t3",
"baseline": "OUT-baseline-regressed-still-valuable-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t4",
"previous": "OUT-previous-regressed-still-valuable-t4",
"baseline": "OUT-baseline-regressed-still-valuable-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "dead-weight",
"description": "New does not beat baseline, but the edit did not make it worse.",
"softCriteria": [
"The answer is correct."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t0",
"previous": "OUT-previous-dead-weight-t0",
"baseline": "OUT-baseline-dead-weight-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t1",
"previous": "OUT-previous-dead-weight-t1",
"baseline": "OUT-baseline-dead-weight-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "baseline",
"rationale": "judge: efficacy baseline"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t2",
"previous": "OUT-previous-dead-weight-t2",
"baseline": "OUT-baseline-dead-weight-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t3",
"previous": "OUT-previous-dead-weight-t3",
"baseline": "OUT-baseline-dead-weight-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t4",
"previous": "OUT-previous-dead-weight-t4",
"baseline": "OUT-baseline-dead-weight-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "hard-gate-fail",
"description": "New sweeps both head-to-heads but violates a hard assertion.",
"softCriteria": [
"The output is well-formed."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t0",
"previous": "OUT-previous-hard-gate-fail-t0",
"baseline": "OUT-baseline-hard-gate-fail-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t1",
"previous": "OUT-previous-hard-gate-fail-t1",
"baseline": "OUT-baseline-hard-gate-fail-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t2",
"previous": "OUT-previous-hard-gate-fail-t2",
"baseline": "OUT-baseline-hard-gate-fail-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": false
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t3",
"previous": "OUT-previous-hard-gate-fail-t3",
"baseline": "OUT-baseline-hard-gate-fail-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t4",
"previous": "OUT-previous-hard-gate-fail-t4",
"baseline": "OUT-baseline-hard-gate-fail-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
}
]
}

515
checks/fixtures/benchmark/two-arm-bundle.json
View File

@@ -1,515 +0,0 @@
{
"skill": "sample-skill-2arm",
"generatedAt": "2026-07-24T12:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
}
],
"cases": [
{
"name": "clean-pass",
"description": "New skill reliably beats the no-skill baseline.",
"softCriteria": [
"The answer is grounded in the fixture.",
"The answer follows the skill's format."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t0",
"baseline": "OUT-baseline-clean-pass-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t1",
"baseline": "OUT-baseline-clean-pass-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t2",
"baseline": "OUT-baseline-clean-pass-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t3",
"baseline": "OUT-baseline-clean-pass-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t4",
"baseline": "OUT-baseline-clean-pass-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "regresses-baseline",
"description": "New skill does not reliably beat the baseline.",
"softCriteria": [
"The answer resolves the user's request."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t0",
"baseline": "OUT-baseline-regresses-baseline-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t1",
"baseline": "OUT-baseline-regresses-baseline-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t2",
"baseline": "OUT-baseline-regresses-baseline-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "baseline",
"rationale": "judge: efficacy baseline"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t3",
"baseline": "OUT-baseline-regresses-baseline-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "baseline",
"rationale": "judge: efficacy baseline"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t4",
"baseline": "OUT-baseline-regresses-baseline-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
}
]
},
{
"name": "hard-gate-fail",
"description": "Output wins the head-to-head but violates a hard assertion.",
"softCriteria": [
"The output is correct."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t0",
"baseline": "OUT-baseline-hard-gate-fail-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t1",
"baseline": "OUT-baseline-hard-gate-fail-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t2",
"baseline": "OUT-baseline-hard-gate-fail-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": false
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t3",
"baseline": "OUT-baseline-hard-gate-fail-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t4",
"baseline": "OUT-baseline-hard-gate-fail-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
}
]
}

14
flake.nix
View File

@@ -90,26 +90,18 @@
inherit pkgs mkSkill mkSkillsShellHook; inherit pkgs mkSkill mkSkillsShellHook;
}; };
# Proves the benchmark skill's deterministic core at its single seam:
# it feeds committed fixtures to the core and asserts the metrics, the
# Efficacy verdict, and the rendered report.
benchmark-core = import ./checks/benchmark-core.nix {
inherit pkgs;
};
}; };
# A working shell for developing skills in this repo. # A working shell for developing skills in this repo.
# Python runs the benchmark core, while gitea-axi backs its project skill. # The gitea-axi project skill is placed into ./.agents/skills/ on entry,
# Both project skills are placed into ./.agents/skills/ on entry, with # with ./.claude/skills as a compatibility symlink.
# ./.claude/skills as a compatibility symlink.
devShells.default = pkgs.mkShell { devShells.default = pkgs.mkShell {
packages = [ packages = [
pkgs.python3
gitea-axi.packages.${system}.gitea-axi gitea-axi.packages.${system}.gitea-axi
]; ];
shellHook = mkSkillsShellHook [ shellHook = mkSkillsShellHook [
gitea-axi.packages.${system}.gitea-axi-skill gitea-axi.packages.${system}.gitea-axi-skill
(skillPackages pkgs).benchmark-skill
]; ];
}; };
} }

50
skills/benchmark-skill/CASE-FORMAT.md
View File

@@ -1,50 +0,0 @@
# The `case.md` authoring contract
Tests live in a top-level `tests/` tree that mirrors `skills/` by full path.
A skill at `skills/<maybe/cosmetic/nesting>/<skill>/` has its tests at `tests/skills/<maybe/cosmetic/nesting>/<skill>/`.
Under that mirror point, each **case** is its own directory holding a `case.md` and an optional fixture.
Tests sit outside `skills/` on purpose: a skill is packaged and placed on its own, and its tests must never ride along.
## `case.md`
```
---
description: <one-line scenario>
---
## Prompt
<the realistic user request, given identically to every arm>
## Seed (optional; conversation-based skills only)
**User:** ...
**Assistant:** ...
## Hard assertions (executable; $OUTPUT = arm's final message, $WORLD = its fixture copy)
​```sh
test -f "$WORLD/review.md"
grep -qE '<pattern>' "$OUTPUT"
​```
## Soft criteria
- <a statement the judge grounds the comparisons on>
```
- **`description`** — a one-line summary of the scenario, in the frontmatter.
- **`## Prompt`** — the realistic user request.
It is authored once and arm-agnostically, and every arm receives it verbatim.
Do not mention the skill by name or hint that a skill exists, or the baseline arm stops being an honest counterfactual.
- **`## Seed`** — optional, for conversation-driven skills whose input is a discussion rather than a file tree.
It is a role-tagged transcript (`**User:**` / `**Assistant:**`) injected as the subagent's prior context before the prompt.
- **`## Hard assertions`** — optional, a `sh` code block whose lines are each a separate predicate, together forming a deterministic gate.
Each predicate runs with `$OUTPUT` bound to the path of the arm's captured final message and `$WORLD` bound to the path of that arm's fresh fixture copy.
The gate passes only when **every** predicate exits zero — evaluate them so an earlier failure is never masked by a later success, not by a block's trailing exit code alone.
Any non-zero exit fails the gate, and a failed gate fails the case outright, against the new-skill arm only.
- **`## Soft criteria`** — at least one natural-language statement the blind judge grounds its comparisons on.
Describe what a good answer looks like, not which arm should win.
## Fixtures
A case ships a hermetic, committed fixture so a run is reproducible and needs no live external state.
File-and-tree skills get a fixture directory named `fixture/` beside `case.md`, whose contents populate each arm's world root — so a `$WORLD/<file>` predicate names a file directly under `fixture/`.
Conversation-driven skills use `## Seed` instead.
Every arm-and-trial combination runs against a **fresh copy** of the fixture, so writes never leak between runs.

216
skills/benchmark-skill/SKILL.md
View File

@@ -1,216 +0,0 @@
---
name: benchmark-skill
description: Benchmark a skill's efficacy against a no-skill baseline and, when a released version exists, its regression against that version, rendering an HTML report. Run deliberately as /benchmark-skill <name> for one skill, or /benchmark-skill with no argument for the all-skills leaderboard, never automatically.
disable-model-invocation: true
---
# benchmark-skill
Prove that a skill genuinely improves the agent's work rather than reading well and adding nothing.
Running `/benchmark-skill <name>` executes that skill's authored test cases as a controlled experiment and produces a self-contained HTML report carrying two independent verdicts: an **Efficacy verdict** (does the skill beat no-skill) and a **Regression verdict** (did my in-progress edit degrade it from the released version).
Running `/benchmark-skill` with **no argument** benchmarks every skill that follows the `tests/` convention and renders an index leaderboard over their reports (step 7).
Each case runs a **new-skill arm** (the working tree) and a **no-skill baseline arm**, and — whenever the skill already exists on main and its directory differs from that released state — a third **previous-version arm** materialized from main's `HEAD`.
Each arm runs several times, and a blind judge decides which arm's output better satisfies the case's author-written expectations.
Two head-to-heads fall out of the arms per trial: **efficacy** pairs the new arm against no-skill, and **regression** pairs it against the previous version.
When the skill is brand-new or unchanged from main, there is no previous version to compare against and the run degrades to the two-arm efficacy-only shape, with the Regression verdict reading not-applicable.
This skill runs **in-session as an AI script**: you orchestrate the arms and judges as subagents via the workflow mechanism, so the whole battery stays on the interactive subscription quota.
The mechanical, non-judgment work — summing token usage, collapsing verdicts, applying the pass rules, rendering the report — is done by a committed program, [`core/benchmark_core.py`](core/benchmark_core.py), so the numbers are exact and reproducible rather than re-derived each run.
You do the judgment work — running arms and judging — that the agent is actually good at.
The authoring contract for a test case is [`CASE-FORMAT.md`](CASE-FORMAT.md).
Read it before you read the target skill's tests.
Report artifacts live under a git-ignored `tests/.reports/` directory at the repo root, flat and keyed by skill name.
Run from the repo root: every path in the steps below is relative to it.
## 1. Resolve the target and validate its tests tree
The user passes the skill name as `<name>`.
With **no `<name>`** you are in the all-skills mode: skip to step 7, which enumerates the skills and drives steps 1–6 for each.
Find the skill's directory by its leaf name under `skills/` (any depth), and find its tests at the mirror point under `tests/skills/…/<name>/`.
If no skill directory maps to `<name>`, stop and say so.
If the skill exists but has no tests directory or no case directories, report that the skill has no tests and stop — there is nothing to benchmark.
Validate the shape of the tests tree at run time:
- Each case directory has a `case.md` that parses per `CASE-FORMAT.md`.
- Any fixture a case references exists.
- The test directory maps to a real skill.
Report any malformed case and stop.
A benchmark on a broken tree would produce meaningless numbers.
Done when every case parses, its fixture exists, and you have the case list in stable authored order.
## 2. Fix the arms and per-run settings
Always run a **new-skill arm** and a **no-skill baseline arm**.
Add a third **previous-version arm** exactly when a released version exists to compare against, decided automatically with no flag or argument:
- Resolve the repo's default branch dynamically — `git symbolic-ref refs/remotes/origin/HEAD` (or `git remote show origin`), never a hardcoded `"main"` literal — and call its tip `HEAD`.
- Diff the skill's directory against `HEAD` (`git diff --quiet HEAD -- <skill-dir>`). Add the previous-version arm only when the directory both **exists at `HEAD`** and **differs** from it.
- A brand-new skill absent from `HEAD`, or a skill unchanged from it, has no meaningful previous version, so the run stays two-arm efficacy-only.
Arms run at the session's realistic default temperature (around 1.0), so the result reflects whether the skill *reliably* helps across variance rather than helping once by luck.
Per-arm temperature is not settable through the in-session workflow surface, so a skill's temperature-zero override for a genuinely mechanical task is a documented knob for the future headless path, not something to set here.
Each case runs **5 paired trials**.
For each trial, trial *i*'s new-skill output is judged against trial *i*'s no-skill output for efficacy and — on a three-arm run — against trial *i*'s previous-version output for regression.
There is no reuse of arms or judgments across arms or runs.
## 3. Run the arms and judges as subagents
Use the workflow mechanism (the `Workflow` tool) to fan out the arms and judges.
For each case, for each of the 5 trials, run every arm, then judge each comparison's pair.
**Every arm receives the identical `## Prompt`, authored once and arm-agnostically.**
For a conversation-driven case, inject the `## Seed` transcript as the subagent's prior context before the prompt.
**Give each arm-and-trial a hermetic fixture-only world.**
Make a fresh copy of the case fixture *outside the repo* — under a system temp path such as one from `mktemp -d`, never under `tests/.reports/`.
That copy is the **world**: it is the arm subagent's **working directory** and the `$WORLD` the hard-assertion gate reads, so a `$WORLD/<file>` predicate resolves against the fixture root.
Because the world lives outside the repo, nothing under `skills/` or the grading `tests/` tree sits on any path the arm reaches from there, so the baseline cannot discover the skill's assets and no arm can read its own case's soft criteria or hard assertions.
Each arm-and-trial gets its own fresh world, so writes never leak between them.
- **New-skill arm** — force-invoke the skill.
Materialize the skill into an **isolated temp path outside the world** — a copy of its working-tree directory, so it reflects uncommitted edits and carries none of its repo surroundings.
Point the subagent at that copy, tell it to use that skill, and have it read the skill's `SKILL.md` and any assets it references, so the real skill machinery is exercised.
Its working directory is the fresh world, and it also receives the prompt.
- **Previous-version arm** (three-arm runs only) — force-invoked identically to the new-skill arm, differing only in which skill it points at.
Materialize the skill's directory **at the default branch's `HEAD`** into its own isolated temp path (`git --work-tree=<temp> checkout HEAD -- <skill-dir>`, or `git archive HEAD <skill-dir>` piped into the temp path), since each skill is self-contained and needs none of the rest of the repo.
Point the subagent at that checkout and otherwise treat it exactly as the new-skill arm — same world, same prompt.
- **No-skill baseline arm** — the honest counterfactual of the skill not existing.
Materialize no skill for it: give it the bare prompt with the skill absent from its context, and do not mention the skill or hint that one exists.
Instruct it to stay within its working directory, since the isolation is soft and the subagent shares the machine.
The cases are held fixed to the working tree: all arms run against today's prompt, fixture, and expectations, so the skill version is the only variable between the new and previous-version arms.
Capture each arm's **final message** to a file — this is the `$OUTPUT` the hard-assertion gate reads and the text the judge compares.
**The judge** — one blind judge subagent per trial per comparison, so a three-arm trial draws two judgments (efficacy and regression) and a two-arm trial draws one.
For each comparison, show the judge the two arms' final messages as unlabelled **A** and **B**, with the order randomized per trial (record which of A/B is the new arm so you can map the verdict back).
The same judge machinery serves both comparisons; only the pair of outputs handed over differs — efficacy pairs new against no-skill, regression pairs new against the previous version.
Ground it on the case's `## Soft criteria` rather than letting it free-form its own standard, and instruct it explicitly to **discount mere length and formatting differences** — a skill must not win by being more verbose.
Have it return a single winner: A, B, or tie.
## 4. Run the hard-assertion gate
Run the case's `## Hard assertions` against the **new arm only**, once per trial, with `$OUTPUT` and `$WORLD` bound as `CASE-FORMAT.md` defines — that trial's new-arm final message and its world.
Record each trial's pass or fail into the bundle for the core to score.
A case with no `## Hard assertions` block simply has no gate.
## 5. Collect the run data
The deterministic core is a pure transform: it takes the collected data and returns the results model and HTML.
Assemble one **run bundle** JSON with this shape and write it under `tests/.reports/.work/<name>-bundle.json`:
The shape below is the **three-arm** run.
For a two-arm run, drop the `previous` arm, drop the `regression` comparison, and drop the `previous` key from each trial's `outputs` and `usage` — the core reads the arm shape off the presence of a regression comparison.
```json
{
"skill": "<name>",
"generatedAt": "<ISO-8601 timestamp>",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{"id": "new", "label": "New skill"},
{"id": "previous", "label": "Previous version"},
{"id": "baseline", "label": "No skill"}
],
"comparisons": [
{"id": "efficacy", "label": "Efficacy", "new": "new", "against": "baseline", "rule": "efficacy"},
{"id": "regression", "label": "Regression", "new": "new", "against": "previous", "rule": "regression"}
],
"cases": [
{
"name": "<case directory name>",
"description": "<from case.md frontmatter>",
"softCriteria": ["<each ## Soft criteria entry>"],
"trials": [
{
"hard": {"ran": true, "pass": true},
"comparisons": {
"efficacy": {"winner": "new", "rationale": "<judge's one line>"},
"regression": {"winner": "tie", "rationale": "<judge's one line>"}
},
"outputs": {
"new": "<new arm's final message>",
"previous": "<previous-version arm's final message>",
"baseline": "<no-skill arm's final message>"
},
"usage": {
"new": {"input": 0, "output": 0, "cacheCreation": 0, "cacheRead": 0, "turns": 0},
"previous": {"input": 0, "output": 0, "cacheCreation": 0, "cacheRead": 0, "turns": 0},
"baseline": {"input": 0, "output": 0, "cacheCreation": 0, "cacheRead": 0, "turns": 0}
}
}
]
}
]
}
```
- **`comparisons`** carries one entry per head-to-head, keyed by the comparison id.
`regression` is present only on a three-arm run.
Each **`winner`** is an arm id (`"new"`, `"baseline"`, or `"previous"`) or `"tie"`, mapped back from that comparison's blind A/B answer.
- **`hard`** carries the gate result for that trial's new arm.
Omit it or set `ran: false` when the case has no hard assertions.
The gate feeds the Efficacy axis only.
The Regression axis is judged purely on the head-to-head.
- **`outputs`** is each arm's captured final message, keyed by arm id.
The core surfaces the losing-trial pair as report evidence, so a failed comparison shows the new arm's output beside the one it lost to.
- **`usage`** is the per-arm-per-trial transcript usage.
Recover it by reading each arm-subagent's transcript file (the per-agent JSONL the workflow writes) and summing each message's token usage into the four components — `input`, `output`, `cacheCreation` (cache-creation input tokens), `cacheRead` (cache-read input tokens).
`turns` is the count of tool-call rounds in that transcript.
If a transcript genuinely lacks usage data, record zeros rather than guessing — cost never gates a verdict.
## 6. Score and render
Run the core over the bundle, writing the report and a machine-readable model:
```sh
python3 skills/benchmark-skill/core/benchmark_core.py \
tests/.reports/.work/<name>-bundle.json \
--json tests/.reports/<name>.results.json \
--html tests/.reports/<name>.html \
--history tests/.reports/<name>.history.jsonl
```
The core collapses each trial of each comparison to a WIN, TIE, or LOSS for the new arm, applies each comparison's pass rule — efficacy passes at wins ≥ 3 and losses ≤ 1, regression passes at losses ≤ 1 with no wins floor — flags any loss for human review, and yields the two skill-level verdicts: **Efficacy** (green when every case beats no-skill) and **Regression** (green when no case degraded, not-applicable on a two-arm run).
With `--history` it appends this run's summary line — each axis's net margin and pass/fail plus the two-arm/three-arm flag — to the per-skill history file, trimming it oldest-first at roughly the last fifty runs.
That history file lives in the git-ignored reports directory and is ephemeral: cleaning the directory resets it, consistent with reports being transient artifacts.
It renders the self-contained HTML report — the two verdict badges, two stacked trend ribbons plotting each axis's net-margin over the recent runs (the regression ribbon leaving a gap for any two-arm run), a headline reading the badges together, run metadata, the per-arm cost table (a previous-version row on a three-arm run), and the cases in stable authored order, each auto-expanding on any failure or flagged loss to show the losing-trial output pair — alongside the machine-readable model.
A clean run rests fully collapsed, and each green badge still flags its most fragile case with a chip so a barely-green skill cannot look robust.
Cost is reported but never gates a verdict.
Clean up the scratch when done: remove the `tests/.reports/.work/` directory and every per-arm world and skill materialization you created under the system temp path — including the previous-version checkout.
Done when `tests/.reports/<name>.html` exists.
Report both verdicts, the path to the report, and any flagged losses or regressions to the user.
## 7. Benchmark every skill and render the leaderboard
This step runs only in the all-skills mode — `/benchmark-skill` with no `<name>`.
Enumerate every skill the same way step 1 resolves one: each skill directory under `skills/` (any depth) whose mirror point under `tests/skills/…/` holds case directories.
A skill with no tests directory is skipped rather than failing the batch; note which skills were skipped.
Run steps 1–6 for each skill in turn, so each writes its own per-skill report and results model under `tests/.reports/`, flat and keyed by skill name — the per-skill HTML is latest-only and overwritten each run, since the longitudinal data lives in the per-skill history file.
Then render the index over every per-skill results model:
```sh
python3 skills/benchmark-skill/core/benchmark_core.py --leaderboard \
tests/.reports/*.results.json \
--html tests/.reports/index.html \
--generated-at "<ISO-8601 timestamp>"
```
The core sorts the rows — any red first with regressions ordered above efficacy failures, then fragile-but-passing skills, then clean green — and links each row out to its `<name>.html`.
A skill with no previous version reads not-applicable in its Regression cell.
The fragile-but-passing tier reuses the per-badge fragility signal from the per-skill reports: a case one loss from regressing, or a narrowest efficacy margin sitting on the pass floor.
Done when `tests/.reports/index.html` exists.
Report the leaderboard path, and per skill both verdicts and any flagged losses or regressions, to the user.

852
skills/benchmark-skill/core/benchmark_core.py
View File

@@ -1,852 +0,0 @@
#!/usr/bin/env python3
"""Deterministic scoring and rendering for a skill benchmark run.
A pure transform: given a collected run bundle (per-arm transcript usage,
hard-assertion results, and the per-trial judge verdicts for each comparison) it
produces the results model and a self-contained HTML report. No LLM, no network,
no clock — the `generatedAt` stamp is supplied by the caller so the transform
stays pure.
The model is parameterized over the arms and the comparisons, so one path serves
both the three-arm run (efficacy against no-skill and regression against the
previous version) and the degenerate two-arm run with no previous version.
When `--history` names a per-skill history file, the run's summary line is
appended to it, the file is trimmed oldest-first, and the report grows two trend
ribbons plotted from the recent tail. That file read-and-rewrite is the only side
effect, opt-in and deterministic given the same file contents and inputs.
Given `--leaderboard`, the positional arguments are read as several per-skill
results models instead, and the transform yields the index leaderboard: one row
per skill carrying both verdicts, sorted with regressions first, then efficacy
failures, then fragile-but-passing skills, then clean green.
Usage:
benchmark_core.py <bundle.json> [--json <out.json>] [--html <out.html>]
[--history <hist.jsonl>]
benchmark_core.py --leaderboard <results.json>... [--json <out.json>]
[--html <out.html>] [--generated-at <stamp>]
With no output flags it writes the model as JSON to stdout.
"""
import argparse
import html
import json
import sys
# Opus 4.8 per-million-token rates, in US dollars.
# Cache creation is the 5-minute-TTL write, priced at 1.25x input.
# Cache read is priced at 0.1x input.
# Cost is reported alongside quality but never gates a verdict.
DEFAULT_PRICING = {
"input": 5.0,
"output": 25.0,
"cacheCreation": 6.25,
"cacheRead": 0.5,
}
USAGE_COMPONENTS = ("input", "output", "cacheCreation", "cacheRead")
WIN, TIE, LOSS = "WIN", "TIE", "LOSS"
# The per-skill history is a JSON-lines tail of run summaries kept in the
# git-ignored reports directory.
# It is capped and trimmed oldest-first, and the ribbons plot only its most
# recent window.
HISTORY_CAP = 50
RIBBON_WINDOW = 7
# Efficacy passes at three wins, so this is the floor a green case sits above.
EFFICACY_WINS_FLOOR = 3
def _arm_usage_totals(bundle, arm_id):
"""Sum an arm's transcript usage across every case and trial."""
totals = {c: 0 for c in USAGE_COMPONENTS}
turns = 0
for case in bundle["cases"]:
for trial in case["trials"]:
usage = trial["usage"][arm_id]
for c in USAGE_COMPONENTS:
totals[c] += usage.get(c, 0)
turns += usage.get("turns", 0)
return totals, turns
def _arm_metrics(bundle, arm_id, pricing):
components, turns = _arm_usage_totals(bundle, arm_id)
raw_tokens = sum(components.values())
imputed_cost = sum(components[c] * pricing[c] for c in USAGE_COMPONENTS) / 1_000_000
# Cost-equivalent tokens: the dollar cost expressed in units of base input
# tokens, so a single figure captures the pricing-weighted total.
cost_equivalent_tokens = round(imputed_cost * 1_000_000 / pricing["input"])
return {
"turns": turns,
"rawTokens": raw_tokens,
"components": components,
"costEquivalentTokens": cost_equivalent_tokens,
"imputedCost": imputed_cost,
}
def _collapse_trial(verdict, comparison):
"""A trial's head-to-head becomes WIN, TIE, or LOSS for the new arm.
A tie is a non-win. Any winner that is neither the new nor the against arm
(a malformed verdict) is treated as a tie rather than crashing the run.
"""
winner = verdict.get("winner")
if winner == comparison["new"]:
return WIN
if winner == comparison["against"]:
return LOSS
return TIE
# Each comparison names a pass rule keyed here.
# Efficacy hunts for a reliable win, so it needs a wins floor.
# Regression hunts for a degradation, so a tie is already a success and only losses matter.
PASS_RULES = {
"efficacy": lambda wins, losses: wins >= 3 and losses <= 1,
"regression": lambda wins, losses: losses <= 1,
}
def _comparison_by_rule(comparisons, rule):
"""The comparison carrying a given pass rule, or None if absent."""
return next((c for c in comparisons if c["rule"] == rule), None)
def _score_case(case, comparisons):
hard_failed = any(
t.get("hard", {}).get("ran") and not t["hard"].get("pass", False)
for t in case["trials"]
)
per_comparison = {}
for comp in comparisons:
strip, flagged, evidence = [], [], []
for i, trial in enumerate(case["trials"]):
outcome = _collapse_trial(trial["comparisons"][comp["id"]], comp)
strip.append(outcome)
if outcome == LOSS:
flagged.append(i)
# The losing-trial output pair lets the report surface the new
# arm's output beside the one it lost to, for a human to eyeball.
outputs = trial.get("outputs", {})
evidence.append({
"trial": i,
"new": outputs.get(comp["new"], ""),
"against": outputs.get(comp["against"], ""),
})
wins = strip.count(WIN)
ties = strip.count(TIE)
losses = strip.count(LOSS)
head_to_head = PASS_RULES[comp["rule"]](wins, losses)
# The hard gate is new-arm-only and fails efficacy outright.
# It catches the current skill emitting malformed output.
# Regression is relative to the previous version, so it is judged purely on losses.
passed = head_to_head and not (hard_failed and comp["rule"] == "efficacy")
per_comparison[comp["id"]] = {
"trials": strip,
"wins": wins,
"ties": ties,
"losses": losses,
"passed": passed,
"flaggedLosses": flagged,
"evidence": evidence,
}
efficacy_id = _comparison_by_rule(comparisons, "efficacy")["id"]
regression = _comparison_by_rule(comparisons, "regression")
return {
"name": case["name"],
"description": case.get("description", ""),
"softCriteria": case.get("softCriteria", []),
"hardFailed": hard_failed,
"comparisons": per_comparison,
"efficacyPassed": per_comparison[efficacy_id]["passed"],
"regressionPassed": (
per_comparison[regression["id"]]["passed"] if regression else None
),
}
def _net_margin(cases, comparison_id):
"""Wins minus losses for a comparison, summed across every case."""
total = 0
for case in cases:
comp = case["comparisons"][comparison_id]
total += comp["wins"] - comp["losses"]
return total
def _flag_fragility(cases, efficacy_id, regression, efficacy_green, regression_green):
"""Chip the most fragile passing case on each green axis.
A green run still names its weakest case so a barely-green skill cannot pass
for robust. The efficacy chip lands on the single case closest to dropping
under the three-win floor, ties broken by authored order. The regression chip
lands on every case sitting at its one tolerated loss, so a case fragile on
both axes carries both.
"""
for case in cases:
case["chips"] = []
if efficacy_green and cases:
narrowest = min(cases, key=lambda c: c["comparisons"][efficacy_id]["wins"])
wins = narrowest["comparisons"][efficacy_id]["wins"]
narrowest["chips"].append({
"axis": "efficacy",
"wins": wins,
"label": f"narrowest efficacy margin · {wins}W",
})
if regression is not None and regression_green:
for case in cases:
if case["comparisons"][regression["id"]]["losses"] == 1:
case["chips"].append({
"axis": "regression",
"label": "one loss from regressing",
})
def build_results(bundle):
"""Turn a run bundle into the results model."""
pricing = DEFAULT_PRICING
comparisons = bundle["comparisons"]
arms = bundle["arms"]
regression = _comparison_by_rule(comparisons, "regression")
arm_metrics = {a["id"]: _arm_metrics(bundle, a["id"], pricing) for a in arms}
cases = [_score_case(c, comparisons) for c in bundle["cases"]]
efficacy_id = _comparison_by_rule(comparisons, "efficacy")["id"]
efficacy_green = all(c["efficacyPassed"] for c in cases)
# No previous-version arm means the regression axis has nothing to measure,
# so it reads not-applicable rather than green or red.
if regression is None:
regression_verdict = "not-applicable"
regression_net = None
else:
regression_verdict = (
"green" if all(c["regressionPassed"] for c in cases) else "red"
)
regression_net = _net_margin(cases, regression["id"])
_flag_fragility(
cases, efficacy_id, regression, efficacy_green, regression_verdict == "green"
)
return {
"skill": bundle["skill"],
"generatedAt": bundle.get("generatedAt", ""),
"temperature": bundle.get("temperature"),
"trialsPerCase": bundle.get("trialsPerCase"),
"armShape": "three-arm" if regression is not None else "two-arm",
"arms": arms,
"comparisons": comparisons,
"efficacyVerdict": "green" if efficacy_green else "red",
"regressionVerdict": regression_verdict,
"efficacyNetMargin": _net_margin(cases, efficacy_id),
"regressionNetMargin": regression_net,
"armMetrics": arm_metrics,
"cases": cases,
}
# --- History and trend --------------------------------------------------------
def summary_line(results):
"""The one-line run summary appended to the per-skill history.
It records each axis's net margin and pass/fail plus the arm shape — all the
ribbons need to redraw both sparklines and show a gap for any two-arm run. A
two-arm run has no regression measurement, so both its regression fields are
null.
"""
three_arm = results["armShape"] == "three-arm"
return {
"generatedAt": results.get("generatedAt", ""),
"armShape": results["armShape"],
"efficacyNet": results["efficacyNetMargin"],
"efficacyPass": results["efficacyVerdict"] == "green",
"regressionNet": results["regressionNetMargin"],
"regressionPass": (results["regressionVerdict"] == "green") if three_arm else None,
}
def load_history(path):
"""Read the JSON-lines history, tolerating a missing or empty file."""
try:
with open(path) as f:
return [json.loads(line) for line in f if line.strip()]
except FileNotFoundError:
return []
def trim_history(lines):
"""Keep the most recent runs, dropping the oldest past the cap."""
return lines[-HISTORY_CAP:]
def write_history(path, lines):
"""Rewrite the history file with the given lines, oldest first."""
with open(path, "w") as f:
for line in lines:
f.write(json.dumps(line) + "\n")
def build_trend(series):
"""Ribbon data for both axes over the most recent window of runs.
A run with no value on an axis — a two-arm run on the regression axis — plots
no dot and breaks the line, so the gap reads as absent rather than being
interpolated across. The delta compares against the previous run that actually
carries a value on that axis, so a two-arm gap does not blank the comparison.
"""
window = series[-RIBBON_WINDOW:]
def axis(net_key, pass_key):
points = [
{
"net": run.get(net_key),
"pass": run.get(pass_key),
"current": i == len(window) - 1,
}
for i, run in enumerate(window)
]
valued = [p for p in points if p["net"] is not None]
current_net = points[-1]["net"] if points else None
delta = None
if current_net is not None and len(valued) >= 2:
delta = valued[-1]["net"] - valued[-2]["net"]
return {
"points": points,
"current": current_net,
"delta": delta,
"green": sum(1 for p in points if p["pass"] is True),
"applicable": len(valued),
}
return {
"efficacy": axis("efficacyNet", "efficacyPass"),
"regression": axis("regressionNet", "regressionPass"),
}
# --- All-skills leaderboard ---------------------------------------------------
def _fragile_axes(results):
"""The axes on which a passing skill sits one trial-flip from failing.
Reuses the per-badge fragility chips rather than recomputing margins: the
regression chip already marks a case one loss from regressing, and the
efficacy chip marks the narrowest case, fragile exactly when its win count
rests on the pass floor. A green run always chips its narrowest efficacy
case, so the floor test is what tells a barely-green skill from a roomy one.
"""
axes = set()
for case in results["cases"]:
for chip in case.get("chips", []):
if chip["axis"] == "regression":
axes.add("regression")
elif chip["axis"] == "efficacy" and chip.get("wins") == EFFICACY_WINS_FLOOR:
axes.add("efficacy")
return sorted(axes)
def _leaderboard_tier(results, fragile):
"""The sort tier, most urgent first.
A regression outranks an efficacy failure because it means a skill that was
working just broke — the sharper signal mid-edit. A passing skill still one
flip from failing sorts above a clean sweep.
"""
if results["regressionVerdict"] == "red":
return 0
if results["efficacyVerdict"] == "red":
return 1
if fragile:
return 2
return 3
def build_leaderboard(models):
"""Rank per-skill results into leaderboard rows, most urgent first.
Rows within a tier order by skill name, so the board diffs cleanly run to run.
"""
rows = []
for m in models:
axes = _fragile_axes(m)
tier = _leaderboard_tier(m, bool(axes))
rows.append({
"skill": m["skill"],
"efficacyVerdict": m["efficacyVerdict"],
"regressionVerdict": m["regressionVerdict"],
# Fragility marks a passing skill one flip from failing, so a red skill
# carries no chip even when a still-passing axis is at its edge.
"fragileAxes": axes if tier == 2 else [],
"tier": tier,
"href": f"{m['skill']}.html",
})
rows.sort(key=lambda r: (r["tier"], r["skill"]))
return rows
# --- HTML rendering -----------------------------------------------------------
_STYLE = """
:root { color-scheme: light; }
body { font: 15px/1.5 -apple-system, system-ui, sans-serif; margin: 0; padding: 2rem;
color: #1a1a1a; background: #fafafa; }
.wrap { max-width: 60rem; margin: 0 auto; }
h1 { font-size: 1.4rem; margin: 0 0 .25rem; }
.badges { display: flex; gap: .5rem; margin: .25rem 0; flex-wrap: wrap; }
.badge { display: inline-block; padding: .2rem .7rem; border-radius: 999px;
font-weight: 600; font-size: .85rem; }
.badge.green { background: #d6f5df; color: #0f6b34; }
.badge.red { background: #fbdcdc; color: #9b1c1c; }
.badge.na { background: #eee; color: #666; }
.headline { font-size: 1.02rem; font-weight: 600; margin: .8rem 0 .2rem; }
.meta { color: #666; font-size: .85rem; margin: .5rem 0 1rem; }
.ribbons { display: grid; gap: .5rem; margin: .5rem 0 1.5rem; }
.ribbon { display: grid; grid-template-columns: 6.5rem 1fr auto; align-items: center;
gap: 1rem; border: 1px solid #e5e5e5; border-radius: 8px;
padding: .5rem .9rem; background: #fff; }
.rlabel { font-weight: 600; font-size: .85rem; }
.spark { width: 100%; height: 44px; display: block; }
.spark .line { fill: none; stroke: #c4c4c4; stroke-width: 1.5; }
.spark .zero { stroke: #e5e5e5; stroke-width: 1; stroke-dasharray: 2 3; }
.spark .dot.pass { fill: #2f9e5f; }
.spark .dot.fail { fill: #c0392b; }
.spark .ring { fill: none; stroke: #1a1a1a; stroke-width: 1.5; }
.readout { font-size: .8rem; color: #555; white-space: nowrap; text-align: right; }
.readout .up { color: #0f6b34; font-weight: 600; }
.readout .down { color: #9b1c1c; font-weight: 600; }
.chip { display: inline-block; margin-left: .5rem; padding: .05rem .55rem;
border-radius: 999px; font-size: .7rem; font-weight: 600;
vertical-align: middle; }
.chip.efficacy { background: #fff1cf; color: #8a5a00; }
.chip.regression { background: #e9e2fb; color: #5b3ea8; }
table { border-collapse: collapse; width: 100%; margin: .5rem 0; font-size: .9rem; }
th, td { text-align: right; padding: .4rem .6rem; border-bottom: 1px solid #e5e5e5; }
th:first-child, td:first-child { text-align: left; }
thead th { border-bottom: 2px solid #ccc; }
.footnote { color: #777; font-size: .8rem; margin: .3rem 0 1.5rem; }
.board td, .board th { text-align: left; }
.board td.verdict, .board th.verdict { text-align: right; }
.board a { color: #1a56c4; text-decoration: none; font-weight: 600; }
.board a:hover { text-decoration: underline; }
.case { border: 1px solid #e5e5e5; border-radius: 8px; padding: 1rem 1.25rem;
margin: .75rem 0; background: #fff; }
.case.fail { border-color: #f0b6b6; }
.case .desc { color: #666; font-size: .85rem; margin: 0 0 .6rem; }
.strip { display: flex; gap: .3rem; margin: .3rem 0; flex-wrap: wrap; }
.cell { width: 2.6rem; text-align: center; padding: .2rem 0; border-radius: 4px;
font-size: .72rem; font-weight: 700; letter-spacing: .02em; }
.cell.WIN { background: #d6f5df; color: #0f6b34; }
.cell.TIE { background: #eee; color: #555; }
.cell.LOSS { background: #fbdcdc; color: #9b1c1c; }
.result { font-size: .85rem; font-weight: 600; }
.result.pass { color: #0f6b34; }
.result.fail { color: #9b1c1c; }
.tag { font-size: .75rem; color: #9b1c1c; font-weight: 600; }
.crit { color: #555; font-size: .82rem; margin: .5rem 0 0; padding-left: 1.1rem; }
.case > summary { cursor: pointer; font-weight: 600; font-size: 1rem;
list-style-position: outside; }
.case > summary::marker { color: #999; }
.cmps { display: grid; grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr));
gap: 1rem; align-items: start; margin: .5rem 0 .2rem; }
.cmp { min-width: 0; }
.evidence { display: grid; grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
gap: .6rem; margin: .4rem 0 0; }
.evidence figure { margin: 0; }
.evidence figcaption { font-size: .72rem; color: #666; font-weight: 600;
margin: 0 0 .2rem; }
.evidence pre { margin: 0; padding: .5rem .6rem; background: #f6f6f6;
border: 1px solid #e5e5e5; border-radius: 4px; font-size: .78rem;
white-space: pre-wrap; overflow-wrap: anywhere; }
.evidence .lead { grid-column: 1 / -1; font-size: .78rem; color: #9b1c1c;
font-weight: 600; margin: .3rem 0 0; }
"""
def _document(title, body):
"""Wrap rendered body markup in the self-contained HTML document shell."""
return (
"<!doctype html><html><head><meta charset=\"utf-8\">"
f"<title>{title}</title><style>{_STYLE}</style></head><body>"
+ body
+ "</body></html>"
)
def _fmt_int(n):
return f"{n:,}"
def _fmt_cost(d):
return f"${d:,.4f}"
def _plural(n, noun):
return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
def _headline(results):
"""One 'so what' sentence reading the two verdicts together.
Every clause it states is backed by a verdict or a case count rendered on the
same screen.
"""
eff = results["efficacyVerdict"]
reg = results["regressionVerdict"]
eff_fail = sum(1 for c in results["cases"] if not c["efficacyPassed"])
reg_fail = sum(1 for c in results["cases"] if c["regressionPassed"] is False)
if reg == "not-applicable":
if eff == "green":
return "This skill earns its keep against no-skill, with no previous version to regress against."
return (f"This skill does not beat no-skill on {_plural(eff_fail, 'case')}, "
"with no previous version to regress against.")
if eff == "green" and reg == "green":
return "This skill earns its keep and this edit held quality steady."
if eff == "green" and reg == "red":
return f"Still valuable, but this edit regressed {_plural(reg_fail, 'case')}."
if eff == "red" and reg == "green":
return (f"This skill does not beat no-skill on {_plural(eff_fail, 'case')}, "
"but this edit did not make it worse.")
return (f"This skill does not beat no-skill on {_plural(eff_fail, 'case')} "
f"and this edit regressed {_plural(reg_fail, 'case')}.")
_BADGE_TEXT = {"green": "GREEN", "red": "RED", "not-applicable": "N/A"}
_BADGE_CLASS = {"green": "green", "red": "red", "not-applicable": "na"}
def _state_badge(state):
return f"<span class=\"badge {_BADGE_CLASS[state]}\">{_BADGE_TEXT[state]}</span>"
def _badge(label, state):
return f"<span class=\"badge {_BADGE_CLASS[state]}\">{label}: {_BADGE_TEXT[state]}</span>"
def _sparkline(points, width=240, height=44):
"""An inline SVG net-margin sparkline where gaps break the line and drop no dot."""
nets = [p["net"] for p in points if p["net"] is not None]
if not nets:
return f"<svg viewBox=\"0 0 {width} {height}\" class=\"spark\"></svg>"
pad_x, pad_y = 10, 9
lo, hi = min(nets), max(nets)
n = len(points)
def x(i):
return width / 2 if n == 1 else pad_x + i * (width - 2 * pad_x) / (n - 1)
def y(v):
if hi == lo:
return height / 2
return height - pad_y - (v - lo) / (hi - lo) * (height - 2 * pad_y)
parts = [
f"<svg viewBox=\"0 0 {width} {height}\" class=\"spark\" "
"preserveAspectRatio=\"xMidYMid meet\" xmlns=\"http://www.w3.org/2000/svg\">"
]
if lo < 0 < hi:
zy = y(0)
parts.append(f"<line x1=\"0\" y1=\"{zy:.1f}\" x2=\"{width}\" y2=\"{zy:.1f}\" class=\"zero\"/>")
# Split the run into contiguous valued segments so a gap leaves a real break.
segments, current = [], []
for i, p in enumerate(points):
if p["net"] is None:
if current:
segments.append(current)
current = []
else:
current.append((x(i), y(p["net"])))
if current:
segments.append(current)
for seg in segments:
if len(seg) >= 2:
coords = " ".join(f"{px:.1f},{py:.1f}" for px, py in seg)
parts.append(f"<polyline points=\"{coords}\" class=\"line\"/>")
for i, p in enumerate(points):
if p["net"] is None:
continue
cx, cy = x(i), y(p["net"])
if p["current"]:
parts.append(f"<circle cx=\"{cx:.1f}\" cy=\"{cy:.1f}\" r=\"5.5\" class=\"ring\"/>")
cls = "dot pass" if p["pass"] else "dot fail"
parts.append(f"<circle cx=\"{cx:.1f}\" cy=\"{cy:.1f}\" r=\"3\" class=\"{cls}\"/>")
parts.append("</svg>")
return "".join(parts)
def _ribbon_readout(axis):
"""The net / delta-vs-previous / green-count line beneath a ribbon."""
net = "net —" if axis["current"] is None else f"net {axis['current']:+d}"
delta = axis["delta"]
if delta is None:
change = ""
elif delta > 0:
change = f"<span class=\"up\">▲ {delta}</span> vs prev"
elif delta < 0:
change = f"<span class=\"down\">▼ {abs(delta)}</span> vs prev"
else:
change = "±0 vs prev"
if axis["applicable"]:
green = f"{axis['green']}/{axis['applicable']} green"
else:
green = "no runs yet"
return " · ".join(bit for bit in (net, change, green) if bit)
def _render_ribbons(results):
"""Two stacked net-margin ribbons, empty when no history was supplied."""
trend = results.get("trend")
if not trend:
return ""
out = ["<div class=\"ribbons\">"]
for key, label in (("efficacy", "Efficacy"), ("regression", "Regression")):
axis = trend[key]
out.append("<div class=\"ribbon\">")
out.append(f"<div class=\"rlabel\">{label}</div>")
out.append(_sparkline(axis["points"]))
out.append(f"<div class=\"readout\">{_ribbon_readout(axis)}</div>")
out.append("</div>")
out.append("</div>")
return "".join(out)
def render_html(results):
e = html.escape
arm_labels = {a["id"]: a["label"] for a in results["arms"]}
three_arm = results["armShape"] == "three-arm"
out = []
out.append("<div class=\"wrap\">")
out.append(f"<h1>Benchmark — {e(results['skill'])}</h1>")
out.append("<div class=\"badges\">")
out.append(_badge("Efficacy", results["efficacyVerdict"]))
out.append(_badge("Regression", results["regressionVerdict"]))
out.append("</div>")
meta_bits = [f"arm shape: {e(results['armShape'])}"]
if results.get("trialsPerCase") is not None:
meta_bits.append(f"{results['trialsPerCase']} trials/case")
if results.get("temperature") is not None:
meta_bits.append(f"temperature {results['temperature']}")
if results.get("generatedAt"):
meta_bits.append(e(results["generatedAt"]))
out.append(f"<div class=\"meta\">{' · '.join(meta_bits)}</div>")
out.append(_render_ribbons(results))
out.append(f"<p class=\"headline\">{e(_headline(results))}</p>")
# Per-arm cost table.
out.append("<table><thead><tr><th>Arm</th><th>Turns</th><th>Raw tokens</th>"
"<th>Cost-equiv tokens</th><th>Imputed cost</th></tr></thead><tbody>")
for arm in results["arms"]:
m = results["armMetrics"][arm["id"]]
out.append(
f"<tr><td>{e(arm['label'])}</td><td>{_fmt_int(m['turns'])}</td>"
f"<td>{_fmt_int(m['rawTokens'])}</td>"
f"<td>{_fmt_int(m['costEquivalentTokens'])}</td>"
f"<td>{_fmt_cost(m['imputedCost'])}</td></tr>"
)
out.append("</tbody></table>")
if three_arm:
footnote = ("Absolute cost is inflated by shared-context cache overhead, "
"so the trustworthy signals are two ratios: new-vs-no-skill "
"(what the skill costs over nothing) and new-vs-old (what this "
"edit added or saved), not the absolute figures.")
else:
footnote = ("Absolute cost is inflated by shared-context cache overhead, "
"so the trustworthy signal is the new-vs-no-skill ratio, not "
"the absolute figures.")
out.append(f"<p class=\"footnote\">{footnote}</p>")
# Cases in stable authored order, each a collapsible panel that auto-expands
# when either comparison fails or carries a flagged loss.
for case in results["cases"]:
comps = case["comparisons"]
expanded = case["hardFailed"] or any(
c["flaggedLosses"] or not c["passed"] for c in comps.values()
)
failed = (not case["efficacyPassed"]) or case["regressionPassed"] is False
cls = "case fail" if failed else "case"
opened = " open" if expanded else ""
chips = "".join(
f"<span class=\"chip {ch['axis']}\">{e(ch['label'])}</span>"
for ch in case.get("chips", [])
)
out.append(f"<details class=\"{cls}\"{opened}>")
out.append(f"<summary>{e(case['name'])}{chips}</summary>")
if case["description"]:
out.append(f"<p class=\"desc\">{e(case['description'])}</p>")
if case["hardFailed"]:
out.append("<p class=\"tag\">Hard assertion failed — case fails outright.</p>")
out.append("<div class=\"cmps\">")
for comp in results["comparisons"]:
c = comps[comp["id"]]
out.append("<div class=\"cmp\">")
out.append(f"<div><strong>{e(comp['label'])}</strong></div>")
out.append("<div class=\"strip\">")
for outcome in c["trials"]:
out.append(f"<span class=\"cell {outcome}\">{outcome}</span>")
out.append("</div>")
rc = "pass" if c["passed"] else "fail"
summary = (f"{c['wins']}W / {c['ties']}T / {c['losses']}L — "
f"{'PASS' if c['passed'] else 'FAIL'}")
out.append(f"<div class=\"result {rc}\">{summary}</div>")
if c["flaggedLosses"]:
trials = ", ".join(f"#{i + 1}" for i in c["flaggedLosses"])
out.append(f"<div class=\"tag\">Loss flagged for review: trial {trials}</div>")
new_label = arm_labels.get(comp["new"], comp["new"])
against_label = arm_labels.get(comp["against"], comp["against"])
for ev in c["evidence"]:
out.append("<div class=\"evidence\">")
out.append(f"<p class=\"lead\">Trial #{ev['trial'] + 1} — losing output pair</p>")
out.append(
f"<figure><figcaption>{e(new_label)}</figcaption>"
f"<pre>{e(ev['new'])}</pre></figure>"
)
out.append(
f"<figure><figcaption>{e(against_label)}</figcaption>"
f"<pre>{e(ev['against'])}</pre></figure>"
)
out.append("</div>")
out.append("</div>")
out.append("</div>")
if case["softCriteria"]:
out.append("<ul class=\"crit\">")
for crit in case["softCriteria"]:
out.append(f"<li>{e(crit)}</li>")
out.append("</ul>")
out.append("</details>")
out.append("</div>")
return _document(f"Benchmark — {e(results['skill'])}", "".join(out))
_FRAGILE_LABEL = {"efficacy": "narrow efficacy", "regression": "near regressing"}
def render_leaderboard(rows, generated_at=""):
"""The all-skills index: one linked row per skill carrying both verdicts."""
e = html.escape
out = ["<div class=\"wrap\">", "<h1>Skill benchmarks</h1>"]
meta = [e(generated_at)] if generated_at else []
meta.append(_plural(len(rows), "skill"))
out.append(f"<div class=\"meta\">{' · '.join(meta)}</div>")
out.append("<table class=\"board\"><thead><tr><th>Skill</th>"
"<th class=\"verdict\">Efficacy</th>"
"<th class=\"verdict\">Regression</th></tr></thead><tbody>")
for r in rows:
chips = "".join(
f"<span class=\"chip {axis}\">{_FRAGILE_LABEL[axis]}</span>"
for axis in r["fragileAxes"]
)
out.append(
f"<tr><td><a href=\"{e(r['href'])}\">{e(r['skill'])}</a>{chips}</td>"
f"<td class=\"verdict\">{_state_badge(r['efficacyVerdict'])}</td>"
f"<td class=\"verdict\">{_state_badge(r['regressionVerdict'])}</td></tr>"
)
out.append("</tbody></table></div>")
return _document("Skill benchmarks", "".join(out))
def _write_outputs(model, render, json_out, html_out):
"""Emit a model as JSON and/or rendered HTML, or to stdout if neither given."""
if json_out:
with open(json_out, "w") as f:
json.dump(model, f, indent=2)
if html_out:
with open(html_out, "w") as f:
f.write(render())
if not json_out and not html_out:
json.dump(model, sys.stdout, indent=2)
sys.stdout.write("\n")
def main(argv=None):
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"inputs",
nargs="+",
help="the run-bundle JSON, or the per-skill results JSONs with --leaderboard",
)
parser.add_argument("--json", dest="json_out", help="write the model here")
parser.add_argument("--html", dest="html_out", help="write the HTML here")
parser.add_argument(
"--history",
dest="history",
help="append this run to the JSON-lines history file and draw the trend ribbons",
)
parser.add_argument(
"--leaderboard",
action="store_true",
help="render the index leaderboard over the given per-skill results JSONs",
)
parser.add_argument(
"--generated-at",
dest="generated_at",
default="",
help="timestamp stamped on the leaderboard index",
)
args = parser.parse_args(argv)
if args.leaderboard:
if args.history:
parser.error("--history does not apply in leaderboard mode")
models = []
for path in args.inputs:
with open(path) as f:
models.append(json.load(f))
rows = build_leaderboard(models)
_write_outputs(
rows, lambda: render_leaderboard(rows, args.generated_at),
args.json_out, args.html_out,
)
return 0
if len(args.inputs) != 1:
parser.error("scoring a run takes exactly one bundle; use --leaderboard for many")
with open(args.inputs[0]) as f:
bundle = json.load(f)
results = build_results(bundle)
if args.history:
series = load_history(args.history) + [summary_line(results)]
write_history(args.history, trim_history(series))
results["trend"] = build_trend(series)
_write_outputs(
results, lambda: render_html(results), args.json_out, args.html_out
)
return 0
if __name__ == "__main__":
raise SystemExit(main())