alexion/skills
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Remove benchmark skill

Browse Source
This commit is contained in:
alexion
2026-07-31 18:50:42 -04:00
parent 8479ad096c
commit d3bc9a5760
11 changed files with 4 additions and 4184 deletions
Download Patch File Download Diff File Expand all files Collapse all files

358
checks/benchmark-core.nix
View File

@@ -1,358 +0,0 @@
# Feeds the committed run-bundle fixtures through the benchmark skill's
# deterministic core and asserts the per-arm metrics, the per-case and
# skill-level Efficacy and Regression verdicts, and the rendered report — for
# both the three-arm shape and the degenerate two-arm "no previous version"
# shape.
# It also drives the longitudinal layer: the per-skill history append-and-trim,
# the two trend ribbons including the two-arm gap in the regression series, and
# the per-badge fragility chips on a clean run and a green-but-fragile one.
# Last it drives the all-skills leaderboard: the tiered sort, the not-applicable
# Regression cell, and the links out to each per-skill report.
# No LLM runs.
{
pkgs,
}:
let
core = ../skills/benchmark-skill/core/benchmark_core.py;
threeArm = ./fixtures/benchmark/three-arm-bundle.json;
twoArm = ./fixtures/benchmark/two-arm-bundle.json;
clean = ./fixtures/benchmark/clean-bundle.json;
fragile = ./fixtures/benchmark/fragile-bundle.json;
in
pkgs.runCommandLocal "benchmark-core-check"
{
nativeBuildInputs = [ pkgs.python3 ];
inherit core threeArm twoArm clean fragile;
}
''
fail() { echo "FAIL: $1" >&2; exit 1; }
# --- three-arm run --------------------------------------------------------
echo "the core scores a three-arm run and renders its report"
python3 "$core" "$threeArm" --json three.json --html three.html \
|| fail "the core exited non-zero on the three-arm bundle"
echo "the three-arm results model carries both verdicts, net margins, and metrics"
python3 - three.json <<'PY' || fail "a three-arm results-model assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["armShape"] == "three-arm", r["armShape"]
# Efficacy is red: dead-weight fails the wins floor and hard-gate-fail trips
# the gate. Regression is red: regressed-still-valuable loses twice to the
# previous version.
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
assert r["regressionVerdict"] == "red", r["regressionVerdict"]
assert r["efficacyNetMargin"] == 12, r["efficacyNetMargin"]
assert r["regressionNetMargin"] == 3, r["regressionNetMargin"]
new = r["armMetrics"]["new"]
assert new["turns"] == 60, new["turns"]
assert new["rawTokens"] == 228000, new["rawTokens"]
assert new["costEquivalentTokens"] == 246000, new["costEquivalentTokens"]
assert abs(new["imputedCost"] - 1.23) < 1e-9, new["imputedCost"]
base = r["armMetrics"]["baseline"]
assert base["turns"] == 40, base["turns"]
assert base["rawTokens"] == 114000, base["rawTokens"]
assert base["costEquivalentTokens"] == 123000, base["costEquivalentTokens"]
prev = r["armMetrics"]["previous"]
assert prev["turns"] == 60, prev["turns"]
assert prev["rawTokens"] == 174000, prev["rawTokens"]
assert prev["costEquivalentTokens"] == 195500, prev["costEquivalentTokens"]
assert abs(prev["imputedCost"] - 0.9775) < 1e-9, prev["imputedCost"]
# Cost-table rows render in arm order: new-skill, previous-version, no-skill.
assert [a["id"] for a in r["arms"]] == ["new", "previous", "baseline"], \
[a["id"] for a in r["arms"]]
cases = {c["name"]: c for c in r["cases"]}
# Authored order is preserved, never reshuffled by verdict.
assert [c["name"] for c in r["cases"]] == [
"clean-both", "regressed-still-valuable", "dead-weight", "hard-gate-fail"
]
a = cases["clean-both"]
assert a["efficacyPassed"] is True
assert a["regressionPassed"] is True
assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"]
assert a["comparisons"]["regression"]["trials"] == ["TIE", "TIE", "WIN", "TIE", "TIE"]
# Efficacy green but regression red — the exact crossing this feature exists
# to catch: still beats no-skill, yet degraded from the released version.
b = cases["regressed-still-valuable"]
assert b["efficacyPassed"] is True
assert b["regressionPassed"] is False
assert b["comparisons"]["regression"]["losses"] == 2
assert b["comparisons"]["regression"]["flaggedLosses"] == [0, 1]
# Efficacy red but regression green — already dead weight, but the edit did
# not make it worse.
c = cases["dead-weight"]
assert c["efficacyPassed"] is False
assert c["regressionPassed"] is True
assert c["comparisons"]["efficacy"]["wins"] == 0
assert c["comparisons"]["efficacy"]["losses"] == 1
assert c["comparisons"]["efficacy"]["flaggedLosses"] == [2]
assert c["comparisons"]["regression"]["wins"] == 3
# The hard gate fails efficacy outright but does not gate regression, which
# is judged purely on losses.
d = cases["hard-gate-fail"]
assert d["hardFailed"] is True
assert d["efficacyPassed"] is False
assert d["regressionPassed"] is True
assert d["comparisons"]["efficacy"]["wins"] == 5
print("three-arm results-model assertions passed")
PY
echo "the three-arm report shows both badges, the three-row table, and evidence"
grep -q '<!doctype html>' three.html || fail "three-arm report is not self-contained"
grep -q 'Efficacy: RED' three.html || fail "three-arm report is missing the red Efficacy badge"
grep -q 'Regression: RED' three.html || fail "three-arm report is missing the red Regression badge"
grep -qi 'regress' three.html || fail "three-arm report is missing the verdict headline"
grep -q 'Cost-equiv tokens' three.html || fail "three-arm report is missing the cost table"
grep -q 'Previous version' three.html || fail "three-arm report is missing the previous-version row"
grep -q 'shared-context cache' three.html || fail "three-arm report is missing the cache footnote"
grep -q 'new-vs-no-skill' three.html || fail "three-arm footnote omits the new-vs-no-skill ratio"
grep -q 'new-vs-old' three.html || fail "three-arm footnote omits the new-vs-old ratio"
for name in clean-both regressed-still-valuable dead-weight hard-gate-fail; do
grep -q "$name" three.html || fail "three-arm report omits case $name"
done
grep -q 'class="cell LOSS"' three.html || fail "three-arm report is missing a LOSS cell"
grep -q 'Hard assertion failed' three.html || fail "three-arm report does not flag the hard failure"
grep -q ' open>' three.html || fail "three-arm report does not auto-expand a failing case"
grep -q 'class="cmps"' three.html || fail "three-arm report does not lay comparisons side by side"
# Evidence for the failed regression comparison: new vs previous on trial 1.
grep -q 'OUT-new-regressed-still-valuable-t0' three.html || fail "missing new evidence for the regression loss"
grep -q 'OUT-previous-regressed-still-valuable-t0' three.html || fail "missing previous evidence for the regression loss"
# Evidence for the failed efficacy comparison: new vs baseline on trial 3.
grep -q 'OUT-new-dead-weight-t2' three.html || fail "missing new evidence for the efficacy loss"
grep -q 'OUT-baseline-dead-weight-t2' three.html || fail "missing baseline evidence for the efficacy loss"
# A clean, collapsed case emits no losing-trial evidence.
if grep -q 'OUT-new-clean-both' three.html; then fail "a clean case leaked losing-trial evidence"; fi
# --- two-arm run ----------------------------------------------------------
echo "the core scores the degenerate two-arm run"
python3 "$core" "$twoArm" --json two.json --html two.html \
|| fail "the core exited non-zero on the two-arm bundle"
echo "the two-arm results model reads regression as not-applicable"
python3 - two.json <<'PY' || fail "a two-arm results-model assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["armShape"] == "two-arm", r["armShape"]
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
assert r["regressionVerdict"] == "not-applicable", r["regressionVerdict"]
assert r["regressionNetMargin"] is None, r["regressionNetMargin"]
assert r["efficacyNetMargin"] == 9, r["efficacyNetMargin"]
new = r["armMetrics"]["new"]
assert new["turns"] == 45, new["turns"]
assert new["rawTokens"] == 171000, new["rawTokens"]
assert new["costEquivalentTokens"] == 184500, new["costEquivalentTokens"]
assert abs(new["imputedCost"] - 0.9225) < 1e-9, new["imputedCost"]
base = r["armMetrics"]["baseline"]
assert base["turns"] == 30, base["turns"]
assert base["rawTokens"] == 85500, base["rawTokens"]
assert base["costEquivalentTokens"] == 92250, base["costEquivalentTokens"]
assert "previous" not in r["armMetrics"], "two-arm run has no previous arm"
assert [c["name"] for c in r["cases"]] == [
"clean-pass", "regresses-baseline", "hard-gate-fail"
]
for c in r["cases"]:
assert c["regressionPassed"] is None, (c["name"], c["regressionPassed"])
b = {c["name"]: c for c in r["cases"]}["regresses-baseline"]
assert b["efficacyPassed"] is False
assert b["comparisons"]["efficacy"]["losses"] == 2
assert b["comparisons"]["efficacy"]["flaggedLosses"] == [3, 4]
h = {c["name"]: c for c in r["cases"]}["hard-gate-fail"]
assert h["hardFailed"] is True
print("two-arm results-model assertions passed")
PY
echo "the two-arm report reads Regression as not-applicable and drops the previous row"
grep -q 'Efficacy: RED' two.html || fail "two-arm report is missing the Efficacy badge"
grep -q 'Regression: N/A' two.html || fail "two-arm report does not read Regression as not-applicable"
grep -q 'new-vs-no-skill' two.html || fail "two-arm footnote omits the new-vs-no-skill ratio"
if grep -q 'new-vs-old' two.html; then fail "two-arm footnote names a new-vs-old ratio that does not apply"; fi
if grep -q 'Previous version' two.html; then fail "two-arm report shows a previous-version row"; fi
grep -q 'OUT-new-regresses-baseline-t3' two.html || fail "missing new evidence for the two-arm efficacy loss"
grep -q 'OUT-baseline-regresses-baseline-t3' two.html || fail "missing baseline evidence for the two-arm efficacy loss"
# --- clean run: fully collapsed, one narrowest-margin chip, ribbons ---------
echo "the core rests a clean run fully collapsed with a lone narrowest-margin chip"
python3 "$core" "$clean" --json clean.json --html clean.html --history clean.history.jsonl \
|| fail "the core exited non-zero on the clean bundle"
python3 - clean.json <<'PY' || fail "a clean-run assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["efficacyVerdict"] == "green", r["efficacyVerdict"]
assert r["regressionVerdict"] == "green", r["regressionVerdict"]
cases = {c["name"]: c for c in r["cases"]}
# The narrowest efficacy case (fewest wins) carries the chip, the roomy one does not.
assert [ch["axis"] for ch in cases["narrowest"]["chips"]] == ["efficacy"], cases["narrowest"]["chips"]
assert cases["roomy"]["chips"] == [], cases["roomy"]["chips"]
# The efficacy chip carries the case's win count, not merely its label text.
assert cases["narrowest"]["chips"][0]["wins"] == 3, cases["narrowest"]["chips"]
print("clean-run assertions passed")
PY
if grep -q ' open>' clean.html; then fail "a clean run did not rest fully collapsed"; fi
grep -q 'class="chip efficacy"' clean.html || fail "clean run omits the narrowest-margin efficacy chip"
if grep -q 'class="chip regression"' clean.html; then fail "clean run shows a spurious regression chip"; fi
grep -q 'class="ribbons"' clean.html || fail "clean run omits the trend ribbons"
grep -q 'class="spark"' clean.html || fail "clean run omits a sparkline"
grep -q 'class="ring"' clean.html || fail "clean run does not ring the current run"
# --- history append-and-trim and the two-ribbon window with a two-arm gap ---
echo "the core appends to and trims the per-skill history and plots the ribbon window"
python3 - seed.history.jsonl <<'PY' || fail "seeding the history fixture failed"
import json, sys
# 55 prior runs, all three-arm except the most recent, which is two-arm and so
# must plot a gap on the regression axis once it lands inside the 7-run window.
lines = []
for i in range(55):
two = i == 54
lines.append({
"generatedAt": f"seed-{i}",
"armShape": "two-arm" if two else "three-arm",
"efficacyNet": i % 5,
"efficacyPass": True,
"regressionNet": None if two else i % 3,
"regressionPass": None if two else True,
})
open(sys.argv[1], "w").write("".join(json.dumps(l) + "\n" for l in lines))
PY
python3 "$core" "$clean" --json trend.json --history seed.history.jsonl \
|| fail "the core exited non-zero on the seeded-history run"
python3 - seed.history.jsonl trend.json <<'PY' || fail "a history/trend assertion failed"
import json, sys
lines = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
# 55 prior + this run = 56, trimmed oldest-first back to the cap of 50.
assert len(lines) == 50, len(lines)
gens = [l["generatedAt"] for l in lines]
assert "seed-0" not in gens and "seed-5" not in gens, "oldest runs were not trimmed"
assert "seed-6" in gens and "seed-54" in gens, "recent runs were wrongly trimmed"
assert lines[-1]["armShape"] == "three-arm", lines[-1]
assert lines[-1]["efficacyNet"] == 8 and lines[-1]["regressionNet"] == 1, lines[-1]
t = json.load(open(sys.argv[2]))["trend"]
eff, reg = t["efficacy"]["points"], t["regression"]["points"]
assert len(eff) == 7 and len(reg) == 7, (len(eff), len(reg))
# Efficacy has a value every run, while regression breaks at the two-arm run.
assert all(p["net"] is not None for p in eff), "efficacy series should have no gaps"
assert eff[-1]["current"] and reg[-1]["current"], "the current run is ringed on both axes"
assert reg[5]["net"] is None, [p["net"] for p in reg]
assert reg[-1]["net"] is not None, "the current three-arm run has a regression dot"
assert t["regression"]["applicable"] == 6, t["regression"]["applicable"]
print("history/trend assertions passed")
PY
# --- green-but-fragile run: independent per-badge chips, both on one case ----
echo "the core chips each fragility axis independently, and both on a doubly-fragile case"
python3 "$core" "$fragile" --json fragile.json --html fragile.html \
|| fail "the core exited non-zero on the fragile bundle"
python3 - fragile.json <<'PY' || fail "a fragility-chip assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["efficacyVerdict"] == "green" and r["regressionVerdict"] == "green"
cases = {c["name"]: c for c in r["cases"]}
# One case is both the narrowest efficacy margin and at exactly one regression
# loss, so it carries both chips.
# The others carry at most their own axis.
assert {ch["axis"] for ch in cases["both"]["chips"]} == {"efficacy", "regression"}, cases["both"]["chips"]
assert [ch["axis"] for ch in cases["reg-only"]["chips"]] == ["regression"], cases["reg-only"]["chips"]
assert cases["none"]["chips"] == [], cases["none"]["chips"]
print("fragility-chip assertions passed")
PY
grep -q 'class="chip efficacy"' fragile.html || fail "fragile run omits the efficacy chip"
grep -q 'class="chip regression"' fragile.html || fail "fragile run omits the regression chip"
# --- all-skills leaderboard: tiered sort, the N/A cell, and per-skill links ---
echo "the core ranks per-skill results into a tiered index leaderboard"
# A synthetic clean-green skill whose narrowest efficacy margin sits above the
# floor, so it is the one input that reaches the clean-green tier. The four
# scored models above supply the other tiers: three-arm regresses (tier 0),
# two-arm fails efficacy with regression not-applicable (tier 1), and the
# clean and fragile green runs are both one flip from failing (tier 2).
python3 - robust.results.json leaky.results.json <<'PY' || fail "writing the leaderboard fixtures failed"
import json, sys
# A clean-green skill whose narrowest efficacy margin clears the floor, the
# one input that reaches the clean-green tier.
json.dump({
"skill": "robust-skill",
"efficacyVerdict": "green",
"regressionVerdict": "green",
"cases": [{"chips": [{"axis": "efficacy", "wins": 5}]}],
}, open(sys.argv[1], "w"))
# An efficacy-red skill whose still-green regression axis sits one loss from
# regressing, so it carries a regression chip yet belongs in the red tier.
json.dump({
"skill": "leaky-skill",
"efficacyVerdict": "red",
"regressionVerdict": "green",
"cases": [{"chips": [{"axis": "regression"}]}],
}, open(sys.argv[2], "w"))
PY
python3 "$core" --leaderboard \
three.json two.json clean.json fragile.json robust.results.json leaky.results.json \
--json board.json --html board.html \
|| fail "the core exited non-zero rendering the leaderboard"
python3 - board.json <<'PY' || fail "a leaderboard assertion failed"
import json, sys
rows = json.load(open(sys.argv[1]))
# Any red first — regressions above efficacy failures, efficacy failures
# ordered by name — then fragile-passing by name, then clean green.
assert [r["skill"] for r in rows] == [
"sample-skill", "leaky-skill", "sample-skill-2arm",
"clean-skill", "fragile-skill", "robust-skill",
], [r["skill"] for r in rows]
assert [r["tier"] for r in rows] == [0, 1, 1, 2, 2, 3], [r["tier"] for r in rows]
by = {r["skill"]: r for r in rows}
# The regressed skill outranks the merely-efficacy-failed ones.
assert by["sample-skill"]["regressionVerdict"] == "red"
# A skill with no previous version reads not-applicable in its Regression cell.
assert by["sample-skill-2arm"]["regressionVerdict"] == "not-applicable"
assert by["sample-skill-2arm"]["efficacyVerdict"] == "red"
# The fragile tier reuses the per-badge chips: a floor-bound efficacy margin,
# a case one loss from regressing, or both.
assert by["clean-skill"]["fragileAxes"] == ["efficacy"], by["clean-skill"]
assert by["fragile-skill"]["fragileAxes"] == ["efficacy", "regression"], by["fragile-skill"]
assert by["robust-skill"]["fragileAxes"] == [], by["robust-skill"]
# A red skill carries no fragility chip even when a still-passing axis is at
# its edge, so the chip stays the marker of the fragile-but-passing tier.
assert by["leaky-skill"]["fragileAxes"] == [], by["leaky-skill"]
# Every row links out to its per-skill report, keyed by skill name.
for r in rows:
assert r["href"] == r["skill"] + ".html", r
print("leaderboard assertions passed")
PY
echo "the leaderboard is self-contained and links out to each per-skill report"
grep -q '<!doctype html>' board.html || fail "the leaderboard is not self-contained"
for skill in sample-skill sample-skill-2arm clean-skill fragile-skill robust-skill leaky-skill; do
grep -q "href=\"$skill.html\"" board.html || fail "the leaderboard omits the link to $skill"
done
grep -q '>N/A<' board.html || fail "the leaderboard omits the not-applicable Regression cell"
grep -q 'near regressing' board.html || fail "the leaderboard omits the regression fragility chip"
grep -q 'narrow efficacy' board.html || fail "the leaderboard omits the efficacy fragility chip"
touch "$out"
''

496
checks/fixtures/benchmark/clean-bundle.json
View File

@@ -1,496 +0,0 @@
{
"skill": "clean-skill",
"generatedAt": "2026-07-24T10:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "narrowest",
"description": "narrowest scenario",
"softCriteria": [
"a good answer for narrowest"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t0",
"previous": "OUT-previous-narrowest-t0",
"baseline": "OUT-baseline-narrowest-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t1",
"previous": "OUT-previous-narrowest-t1",
"baseline": "OUT-baseline-narrowest-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t2",
"previous": "OUT-previous-narrowest-t2",
"baseline": "OUT-baseline-narrowest-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t3",
"previous": "OUT-previous-narrowest-t3",
"baseline": "OUT-baseline-narrowest-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t4",
"previous": "OUT-previous-narrowest-t4",
"baseline": "OUT-baseline-narrowest-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "roomy",
"description": "roomy scenario",
"softCriteria": [
"a good answer for roomy"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "new",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t0",
"previous": "OUT-previous-roomy-t0",
"baseline": "OUT-baseline-roomy-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t1",
"previous": "OUT-previous-roomy-t1",
"baseline": "OUT-baseline-roomy-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t2",
"previous": "OUT-previous-roomy-t2",
"baseline": "OUT-baseline-roomy-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t3",
"previous": "OUT-previous-roomy-t3",
"baseline": "OUT-baseline-roomy-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t4",
"previous": "OUT-previous-roomy-t4",
"baseline": "OUT-baseline-roomy-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
}
]
}

725
checks/fixtures/benchmark/fragile-bundle.json
View File

@@ -1,725 +0,0 @@
{
"skill": "fragile-skill",
"generatedAt": "2026-07-24T11:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "both",
"description": "both scenario",
"softCriteria": [
"a good answer for both"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "new",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t0",
"previous": "OUT-previous-both-t0",
"baseline": "OUT-baseline-both-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t1",
"previous": "OUT-previous-both-t1",
"baseline": "OUT-baseline-both-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t2",
"previous": "OUT-previous-both-t2",
"baseline": "OUT-baseline-both-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t3",
"previous": "OUT-previous-both-t3",
"baseline": "OUT-baseline-both-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "previous",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t4",
"previous": "OUT-previous-both-t4",
"baseline": "OUT-baseline-both-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "reg-only",
"description": "reg-only scenario",
"softCriteria": [
"a good answer for reg-only"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t0",
"previous": "OUT-previous-reg-only-t0",
"baseline": "OUT-baseline-reg-only-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t1",
"previous": "OUT-previous-reg-only-t1",
"baseline": "OUT-baseline-reg-only-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t2",
"previous": "OUT-previous-reg-only-t2",
"baseline": "OUT-baseline-reg-only-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t3",
"previous": "OUT-previous-reg-only-t3",
"baseline": "OUT-baseline-reg-only-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "previous",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t4",
"previous": "OUT-previous-reg-only-t4",
"baseline": "OUT-baseline-reg-only-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "none",
"description": "none scenario",
"softCriteria": [
"a good answer for none"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t0",
"previous": "OUT-previous-none-t0",
"baseline": "OUT-baseline-none-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t1",
"previous": "OUT-previous-none-t1",
"baseline": "OUT-baseline-none-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t2",
"previous": "OUT-previous-none-t2",
"baseline": "OUT-baseline-none-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t3",
"previous": "OUT-previous-none-t3",
"baseline": "OUT-baseline-none-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t4",
"previous": "OUT-previous-none-t4",
"baseline": "OUT-baseline-none-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
}
]
}

954
checks/fixtures/benchmark/three-arm-bundle.json
View File

@@ -1,954 +0,0 @@
{
"skill": "sample-skill",
"generatedAt": "2026-07-24T12:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "clean-both",
"description": "New beats baseline and holds steady versus the previous version.",
"softCriteria": [
"The answer is grounded in the fixture."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t0",
"previous": "OUT-previous-clean-both-t0",
"baseline": "OUT-baseline-clean-both-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t1",
"previous": "OUT-previous-clean-both-t1",
"baseline": "OUT-baseline-clean-both-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-clean-both-t2",
"previous": "OUT-previous-clean-both-t2",
"baseline": "OUT-baseline-clean-both-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t3",
"previous": "OUT-previous-clean-both-t3",
"baseline": "OUT-baseline-clean-both-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-clean-both-t4",
"previous": "OUT-previous-clean-both-t4",
"baseline": "OUT-baseline-clean-both-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "regressed-still-valuable",
"description": "New still beats baseline but has degraded from the previous version.",
"softCriteria": [
"The answer resolves the user's request."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "previous",
"rationale": "judge: regression previous"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t0",
"previous": "OUT-previous-regressed-still-valuable-t0",
"baseline": "OUT-baseline-regressed-still-valuable-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "previous",
"rationale": "judge: regression previous"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t1",
"previous": "OUT-previous-regressed-still-valuable-t1",
"baseline": "OUT-baseline-regressed-still-valuable-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t2",
"previous": "OUT-previous-regressed-still-valuable-t2",
"baseline": "OUT-baseline-regressed-still-valuable-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t3",
"previous": "OUT-previous-regressed-still-valuable-t3",
"baseline": "OUT-baseline-regressed-still-valuable-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-regressed-still-valuable-t4",
"previous": "OUT-previous-regressed-still-valuable-t4",
"baseline": "OUT-baseline-regressed-still-valuable-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "dead-weight",
"description": "New does not beat baseline, but the edit did not make it worse.",
"softCriteria": [
"The answer is correct."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t0",
"previous": "OUT-previous-dead-weight-t0",
"baseline": "OUT-baseline-dead-weight-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t1",
"previous": "OUT-previous-dead-weight-t1",
"baseline": "OUT-baseline-dead-weight-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "baseline",
"rationale": "judge: efficacy baseline"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t2",
"previous": "OUT-previous-dead-weight-t2",
"baseline": "OUT-baseline-dead-weight-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "new",
"rationale": "judge: regression new"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t3",
"previous": "OUT-previous-dead-weight-t3",
"baseline": "OUT-baseline-dead-weight-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-dead-weight-t4",
"previous": "OUT-previous-dead-weight-t4",
"baseline": "OUT-baseline-dead-weight-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "hard-gate-fail",
"description": "New sweeps both head-to-heads but violates a hard assertion.",
"softCriteria": [
"The output is well-formed."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t0",
"previous": "OUT-previous-hard-gate-fail-t0",
"baseline": "OUT-baseline-hard-gate-fail-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t1",
"previous": "OUT-previous-hard-gate-fail-t1",
"baseline": "OUT-baseline-hard-gate-fail-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t2",
"previous": "OUT-previous-hard-gate-fail-t2",
"baseline": "OUT-baseline-hard-gate-fail-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": false
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t3",
"previous": "OUT-previous-hard-gate-fail-t3",
"baseline": "OUT-baseline-hard-gate-fail-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
},
"regression": {
"winner": "tie",
"rationale": "judge: regression tie"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t4",
"previous": "OUT-previous-hard-gate-fail-t4",
"baseline": "OUT-baseline-hard-gate-fail-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
},
"previous": {
"input": 800,
"output": 1600,
"cacheCreation": 300,
"cacheRead": 6000,
"turns": 3
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
}
]
}

515
checks/fixtures/benchmark/two-arm-bundle.json
View File

@@ -1,515 +0,0 @@
{
"skill": "sample-skill-2arm",
"generatedAt": "2026-07-24T12:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
}
],
"cases": [
{
"name": "clean-pass",
"description": "New skill reliably beats the no-skill baseline.",
"softCriteria": [
"The answer is grounded in the fixture.",
"The answer follows the skill's format."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t0",
"baseline": "OUT-baseline-clean-pass-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t1",
"baseline": "OUT-baseline-clean-pass-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t2",
"baseline": "OUT-baseline-clean-pass-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t3",
"baseline": "OUT-baseline-clean-pass-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
}
},
"outputs": {
"new": "OUT-new-clean-pass-t4",
"baseline": "OUT-baseline-clean-pass-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
},
{
"name": "regresses-baseline",
"description": "New skill does not reliably beat the baseline.",
"softCriteria": [
"The answer resolves the user's request."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t0",
"baseline": "OUT-baseline-regresses-baseline-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t1",
"baseline": "OUT-baseline-regresses-baseline-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "judge: efficacy tie"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t2",
"baseline": "OUT-baseline-regresses-baseline-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "baseline",
"rationale": "judge: efficacy baseline"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t3",
"baseline": "OUT-baseline-regresses-baseline-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
},
{
"comparisons": {
"efficacy": {
"winner": "baseline",
"rationale": "judge: efficacy baseline"
}
},
"outputs": {
"new": "OUT-new-regresses-baseline-t4",
"baseline": "OUT-baseline-regresses-baseline-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
}
}
]
},
{
"name": "hard-gate-fail",
"description": "Output wins the head-to-head but violates a hard assertion.",
"softCriteria": [
"The output is correct."
],
"trials": [
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t0",
"baseline": "OUT-baseline-hard-gate-fail-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t1",
"baseline": "OUT-baseline-hard-gate-fail-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t2",
"baseline": "OUT-baseline-hard-gate-fail-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": false
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t3",
"baseline": "OUT-baseline-hard-gate-fail-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
},
{
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "judge: efficacy new"
}
},
"outputs": {
"new": "OUT-new-hard-gate-fail-t4",
"baseline": "OUT-baseline-hard-gate-fail-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 2000,
"cacheCreation": 400,
"cacheRead": 8000,
"turns": 3
},
"baseline": {
"input": 500,
"output": 1000,
"cacheCreation": 200,
"cacheRead": 4000,
"turns": 2
}
},
"hard": {
"ran": true,
"pass": true
}
}
]
}
]
}