diff --git a/.claude/tasks/0007-all-skills-leaderboard.md b/.claude/tasks/0007-all-skills-leaderboard.md new file mode 100644 index 0000000..88efcec --- /dev/null +++ b/.claude/tasks/0007-all-skills-leaderboard.md @@ -0,0 +1,46 @@ +--- +spec: skill-benchmarking +blocked-by: 0006-trend-history-and-ribbons +--- + +## What to build + +The batch view: `/benchmark-skill` with no argument benchmarks every skill and produces an index leaderboard, so an author knows at a glance which skills are green and which regressed. + +Invoked with no argument, the runner benchmarks every skill that follows the `tests/` convention and renders an index leaderboard. +Each skill row carries both badges, Efficacy and Regression, and a skill with no previous version reads not-applicable in its Regression cell. +The sort promotes any red first, with regressions ordered above efficacy failures — a regression means "you just broke something that was working," the more urgent signal while iterating — then fragile-but-passing skills, then clean green. +The fragile-but-passing tier reuses the per-badge fragility signal introduced with the trend layer (0006) rather than recomputing it here. +The leaderboard links out to the separate per-skill report files produced by a single-skill run. + +The per-skill report files and the index live flat under the `tests/.reports/` directory (its git-ignore established in 0004), keyed by unique skill name, since skill names are globally unique in this repo. +Per-skill HTML is latest-only and overwritten each run, because the longitudinal data already lives in the per-skill history file. + +The core's fixture unit test is extended to cover the leaderboard rendering: the sort order across a mix of regressed, efficacy-failed, fragile, and clean skills, the not-applicable Regression cell for a skill with no previous version, and the links to per-skill files. + +## Acceptance criteria + +- [x] `/benchmark-skill` with no argument benchmarks every skill following the `tests/` convention and renders an index leaderboard. +- [x] Each leaderboard row carries both the Efficacy and Regression badges; a skill with no previous version reads not-applicable in its Regression cell. +- [x] The sort promotes any red first with regressions above efficacy failures, then fragile-but-passing skills, then clean green. +- [x] The leaderboard links out to the separate per-skill report files. +- [x] The per-skill report files and the leaderboard index live flat under `tests/.reports/` (its git-ignore established in 0004), keyed by unique skill name. +- [x] Per-skill HTML is latest-only and overwritten each run. +- [x] The core's fixture unit test covers the leaderboard sort order, the not-applicable Regression cell, and the per-skill links. + +## Implementation Notes + +The leaderboard is a second pure transform in the deterministic core, `core/benchmark_core.py`, fully covered by the extended `checks/benchmark-core.nix` fixture test. +The in-session orchestration half — enumerate every skill with tests, run steps 1–6 for each, then render the index — is documented in `SKILL.md` step 7 rather than unit-tested, following the 0006 split where the workflow half is established by running the harness, not the fixture test. +Per the invocation, the benchmark harness itself was not run. + +- **The leaderboard consumes per-skill results JSONs, not bundles, so the core stays a pure transform.** + The single-skill flow already writes `tests/.reports/.results.json`; the batch flow feeds every one of those to `--leaderboard`, which sorts and renders the index to `tests/.reports/index.html`. +- **The CLI gains a `--leaderboard` mode that reuses `--json`/`--html`.** + The positional argument was widened from a single `bundle` to `inputs` (`nargs="+"`) with an explicit count guard, so the single-skill contract `core.py --json … --html …` is unchanged and every existing call site still works. +- **The fragile-but-passing tier reuses the 0006 per-badge chips rather than recomputing margins.** + The regression chip already means "one loss from regressing." A green run always chips its narrowest efficacy case, so chip presence alone cannot tell a barely-green skill from a roomy one; the efficacy chip therefore now carries its win count, and the leaderboard reads that count against the existing `EFFICACY_WINS_FLOOR` to decide efficacy fragility — the spec's own definition, "closest to dropping under three wins." +- **Fragility is scoped to the fragile-but-passing tier (review finding).** + A red skill carries no fragility chip even when a still-passing axis sits at its edge, so the chip stays the marker of tier 2 rather than leaking onto a red row. +- **Two leaderboard fixtures are authored inline in the check, alongside the four scored models.** + No committed bundle reaches the clean-green tier (the clean bundle's narrowest efficacy case sits on the floor, so it is itself fragile) or the efficacy-red-with-a-still-fragile-regression-axis crossing, so `robust-skill` and `leaky-skill` are synthesized as small results JSONs the way 0006 synthesized its 55-line history seed. diff --git a/checks/benchmark-core.nix b/checks/benchmark-core.nix index b85dbce..6d363f1 100644 --- a/checks/benchmark-core.nix +++ b/checks/benchmark-core.nix @@ -6,6 +6,8 @@ # It also drives the longitudinal layer: the per-skill history append-and-trim, # the two trend ribbons including the two-arm gap in the regression series, and # the per-badge fragility chips on a clean run and a green-but-fragile one. +# Last it drives the all-skills leaderboard: the tiered sort, the not-applicable +# Regression cell, and the links out to each per-skill report. # No LLM runs. { pkgs, @@ -199,6 +201,8 @@ pkgs.runCommandLocal "benchmark-core-check" # The narrowest efficacy case (fewest wins) carries the chip, the roomy one does not. assert [ch["axis"] for ch in cases["narrowest"]["chips"]] == ["efficacy"], cases["narrowest"]["chips"] assert cases["roomy"]["chips"] == [], cases["roomy"]["chips"] + # The efficacy chip carries the case's win count, not merely its label text. + assert cases["narrowest"]["chips"][0]["wins"] == 3, cases["narrowest"]["chips"] print("clean-run assertions passed") PY @@ -277,5 +281,78 @@ pkgs.runCommandLocal "benchmark-core-check" grep -q 'class="chip efficacy"' fragile.html || fail "fragile run omits the efficacy chip" grep -q 'class="chip regression"' fragile.html || fail "fragile run omits the regression chip" + # --- all-skills leaderboard: tiered sort, the N/A cell, and per-skill links --- + echo "the core ranks per-skill results into a tiered index leaderboard" + # A synthetic clean-green skill whose narrowest efficacy margin sits above the + # floor, so it is the one input that reaches the clean-green tier. The four + # scored models above supply the other tiers: three-arm regresses (tier 0), + # two-arm fails efficacy with regression not-applicable (tier 1), and the + # clean and fragile green runs are both one flip from failing (tier 2). + python3 - robust.results.json leaky.results.json <<'PY' || fail "writing the leaderboard fixtures failed" + import json, sys + # A clean-green skill whose narrowest efficacy margin clears the floor, the + # one input that reaches the clean-green tier. + json.dump({ + "skill": "robust-skill", + "efficacyVerdict": "green", + "regressionVerdict": "green", + "cases": [{"chips": [{"axis": "efficacy", "wins": 5}]}], + }, open(sys.argv[1], "w")) + # An efficacy-red skill whose still-green regression axis sits one loss from + # regressing, so it carries a regression chip yet belongs in the red tier. + json.dump({ + "skill": "leaky-skill", + "efficacyVerdict": "red", + "regressionVerdict": "green", + "cases": [{"chips": [{"axis": "regression"}]}], + }, open(sys.argv[2], "w")) + PY + + python3 "$core" --leaderboard \ + three.json two.json clean.json fragile.json robust.results.json leaky.results.json \ + --json board.json --html board.html \ + || fail "the core exited non-zero rendering the leaderboard" + + python3 - board.json <<'PY' || fail "a leaderboard assertion failed" + import json, sys + rows = json.load(open(sys.argv[1])) + + # Any red first — regressions above efficacy failures, efficacy failures + # ordered by name — then fragile-passing by name, then clean green. + assert [r["skill"] for r in rows] == [ + "sample-skill", "leaky-skill", "sample-skill-2arm", + "clean-skill", "fragile-skill", "robust-skill", + ], [r["skill"] for r in rows] + assert [r["tier"] for r in rows] == [0, 1, 1, 2, 2, 3], [r["tier"] for r in rows] + + by = {r["skill"]: r for r in rows} + # The regressed skill outranks the merely-efficacy-failed ones. + assert by["sample-skill"]["regressionVerdict"] == "red" + # A skill with no previous version reads not-applicable in its Regression cell. + assert by["sample-skill-2arm"]["regressionVerdict"] == "not-applicable" + assert by["sample-skill-2arm"]["efficacyVerdict"] == "red" + # The fragile tier reuses the per-badge chips: a floor-bound efficacy margin, + # a case one loss from regressing, or both. + assert by["clean-skill"]["fragileAxes"] == ["efficacy"], by["clean-skill"] + assert by["fragile-skill"]["fragileAxes"] == ["efficacy", "regression"], by["fragile-skill"] + assert by["robust-skill"]["fragileAxes"] == [], by["robust-skill"] + # A red skill carries no fragility chip even when a still-passing axis is at + # its edge, so the chip stays the marker of the fragile-but-passing tier. + assert by["leaky-skill"]["fragileAxes"] == [], by["leaky-skill"] + # Every row links out to its per-skill report, keyed by skill name. + for r in rows: + assert r["href"] == r["skill"] + ".html", r + print("leaderboard assertions passed") + PY + + echo "the leaderboard is self-contained and links out to each per-skill report" + grep -q '' board.html || fail "the leaderboard is not self-contained" + for skill in sample-skill sample-skill-2arm clean-skill fragile-skill robust-skill leaky-skill; do + grep -q "href=\"$skill.html\"" board.html || fail "the leaderboard omits the link to $skill" + done + grep -q '>N/A<' board.html || fail "the leaderboard omits the not-applicable Regression cell" + grep -q 'near regressing' board.html || fail "the leaderboard omits the regression fragility chip" + grep -q 'narrow efficacy' board.html || fail "the leaderboard omits the efficacy fragility chip" + touch "$out" '' diff --git a/skills/benchmark-skill/SKILL.md b/skills/benchmark-skill/SKILL.md index e445431..4f1a583 100644 --- a/skills/benchmark-skill/SKILL.md +++ b/skills/benchmark-skill/SKILL.md @@ -1,6 +1,6 @@ --- name: benchmark-skill -description: Benchmark a skill's efficacy against a no-skill baseline and, when a released version exists, its regression against that version, rendering an HTML report. Run deliberately as /benchmark-skill , never automatically. +description: Benchmark a skill's efficacy against a no-skill baseline and, when a released version exists, its regression against that version, rendering an HTML report. Run deliberately as /benchmark-skill for one skill, or /benchmark-skill with no argument for the all-skills leaderboard, never automatically. disable-model-invocation: true --- @@ -9,6 +9,7 @@ disable-model-invocation: true Prove that a skill genuinely improves the agent's work rather than reading well and adding nothing. Running `/benchmark-skill ` executes that skill's authored test cases as a controlled experiment and produces a self-contained HTML report carrying two independent verdicts: an **Efficacy verdict** (does the skill beat no-skill) and a **Regression verdict** (did my in-progress edit degrade it from the released version). +Running `/benchmark-skill` with **no argument** benchmarks every skill that follows the `tests/` convention and renders an index leaderboard over their reports (step 7). Each case runs a **new-skill arm** (the working tree) and a **no-skill baseline arm**, and — whenever the skill already exists on main and its directory differs from that released state — a third **previous-version arm** materialized from main's `HEAD`. Each arm runs several times, and a blind judge decides which arm's output better satisfies the case's author-written expectations. Two head-to-heads fall out of the arms per trial: **efficacy** pairs the new arm against no-skill, and **regression** pairs it against the previous version. @@ -27,6 +28,7 @@ Run from the repo root: every path in the steps below is relative to it. ## 1. Resolve the target and validate its tests tree The user passes the skill name as ``. +With **no ``** you are in the all-skills mode: skip to step 7, which enumerates the skills and drives steps 1–6 for each. Find the skill's directory by its leaf name under `skills/` (any depth), and find its tests at the mirror point under `tests/skills/…//`. If no skill directory maps to ``, stop and say so. If the skill exists but has no tests directory or no case directories, report that the skill has no tests and stop — there is nothing to benchmark. @@ -188,3 +190,27 @@ Clean up the scratch when done: remove the `tests/.reports/.work/` directory and Done when `tests/.reports/.html` exists. Report both verdicts, the path to the report, and any flagged losses or regressions to the user. + +## 7. Benchmark every skill and render the leaderboard + +This step runs only in the all-skills mode — `/benchmark-skill` with no ``. + +Enumerate every skill the same way step 1 resolves one: each skill directory under `skills/` (any depth) whose mirror point under `tests/skills/…/` holds case directories. +A skill with no tests directory is skipped rather than failing the batch; note which skills were skipped. +Run steps 1–6 for each skill in turn, so each writes its own per-skill report and results model under `tests/.reports/`, flat and keyed by skill name — the per-skill HTML is latest-only and overwritten each run, since the longitudinal data lives in the per-skill history file. + +Then render the index over every per-skill results model: + +```sh +python3 skills/benchmark-skill/core/benchmark_core.py --leaderboard \ + tests/.reports/*.results.json \ + --html tests/.reports/index.html \ + --generated-at "" +``` + +The core sorts the rows — any red first with regressions ordered above efficacy failures, then fragile-but-passing skills, then clean green — and links each row out to its `.html`. +A skill with no previous version reads not-applicable in its Regression cell. +The fragile-but-passing tier reuses the per-badge fragility signal from the per-skill reports: a case one loss from regressing, or a narrowest efficacy margin sitting on the pass floor. + +Done when `tests/.reports/index.html` exists. +Report the leaderboard path, and per skill both verdicts and any flagged losses or regressions, to the user. diff --git a/skills/benchmark-skill/core/benchmark_core.py b/skills/benchmark-skill/core/benchmark_core.py index bc7c53f..eb1b4b3 100644 --- a/skills/benchmark-skill/core/benchmark_core.py +++ b/skills/benchmark-skill/core/benchmark_core.py @@ -16,11 +16,18 @@ appended to it, the file is trimmed oldest-first, and the report grows two trend ribbons plotted from the recent tail. That file read-and-rewrite is the only side effect, opt-in and deterministic given the same file contents and inputs. +Given `--leaderboard`, the positional arguments are read as several per-skill +results models instead, and the transform yields the index leaderboard: one row +per skill carrying both verdicts, sorted with regressions first, then efficacy +failures, then fragile-but-passing skills, then clean green. + Usage: benchmark_core.py [--json ] [--html ] [--history ] + benchmark_core.py --leaderboard ... [--json ] + [--html ] [--generated-at ] -With no output flags it writes the results model as JSON to stdout. +With no output flags it writes the model as JSON to stdout. """ import argparse @@ -193,6 +200,7 @@ def _flag_fragility(cases, efficacy_id, regression, efficacy_green, regression_g wins = narrowest["comparisons"][efficacy_id]["wins"] narrowest["chips"].append({ "axis": "efficacy", + "wins": wins, "label": f"narrowest efficacy margin · {wins}W", }) @@ -331,6 +339,67 @@ def build_trend(series): } +# --- All-skills leaderboard --------------------------------------------------- + + +def _fragile_axes(results): + """The axes on which a passing skill sits one trial-flip from failing. + + Reuses the per-badge fragility chips rather than recomputing margins: the + regression chip already marks a case one loss from regressing, and the + efficacy chip marks the narrowest case, fragile exactly when its win count + rests on the pass floor. A green run always chips its narrowest efficacy + case, so the floor test is what tells a barely-green skill from a roomy one. + """ + axes = set() + for case in results["cases"]: + for chip in case.get("chips", []): + if chip["axis"] == "regression": + axes.add("regression") + elif chip["axis"] == "efficacy" and chip.get("wins") == EFFICACY_WINS_FLOOR: + axes.add("efficacy") + return sorted(axes) + + +def _leaderboard_tier(results, fragile): + """The sort tier, most urgent first. + + A regression outranks an efficacy failure because it means a skill that was + working just broke — the sharper signal mid-edit. A passing skill still one + flip from failing sorts above a clean sweep. + """ + if results["regressionVerdict"] == "red": + return 0 + if results["efficacyVerdict"] == "red": + return 1 + if fragile: + return 2 + return 3 + + +def build_leaderboard(models): + """Rank per-skill results into leaderboard rows, most urgent first. + + Rows within a tier order by skill name, so the board diffs cleanly run to run. + """ + rows = [] + for m in models: + axes = _fragile_axes(m) + tier = _leaderboard_tier(m, bool(axes)) + rows.append({ + "skill": m["skill"], + "efficacyVerdict": m["efficacyVerdict"], + "regressionVerdict": m["regressionVerdict"], + # Fragility marks a passing skill one flip from failing, so a red skill + # carries no chip even when a still-passing axis is at its edge. + "fragileAxes": axes if tier == 2 else [], + "tier": tier, + "href": f"{m['skill']}.html", + }) + rows.sort(key=lambda r: (r["tier"], r["skill"])) + return rows + + # --- HTML rendering ----------------------------------------------------------- _STYLE = """ @@ -371,6 +440,10 @@ th, td { text-align: right; padding: .4rem .6rem; border-bottom: 1px solid #e5e5 th:first-child, td:first-child { text-align: left; } thead th { border-bottom: 2px solid #ccc; } .footnote { color: #777; font-size: .8rem; margin: .3rem 0 1.5rem; } +.board td, .board th { text-align: left; } +.board td.verdict, .board th.verdict { text-align: right; } +.board a { color: #1a56c4; text-decoration: none; font-weight: 600; } +.board a:hover { text-decoration: underline; } .case { border: 1px solid #e5e5e5; border-radius: 8px; padding: 1rem 1.25rem; margin: .75rem 0; background: #fff; } .case.fail { border-color: #f0b6b6; } @@ -405,6 +478,16 @@ thead th { border-bottom: 2px solid #ccc; } """ +def _document(title, body): + """Wrap rendered body markup in the self-contained HTML document shell.""" + return ( + "" + f"{title}" + + body + + "" + ) + + def _fmt_int(n): return f"{n:,}" @@ -444,10 +527,16 @@ def _headline(results): f"and this edit regressed {_plural(reg_fail, 'case')}.") +_BADGE_TEXT = {"green": "GREEN", "red": "RED", "not-applicable": "N/A"} +_BADGE_CLASS = {"green": "green", "red": "red", "not-applicable": "na"} + + +def _state_badge(state): + return f"{_BADGE_TEXT[state]}" + + def _badge(label, state): - text = {"green": "GREEN", "red": "RED", "not-applicable": "N/A"}[state] - cls = {"green": "green", "red": "red", "not-applicable": "na"}[state] - return f"{label}: {text}" + return f"{label}: {_BADGE_TEXT[state]}" def _sparkline(points, width=240, height=44): @@ -653,28 +742,96 @@ def render_html(results): out.append("") - title = f"Benchmark — {e(results['skill'])}" - return ( - "" - f"{title}" - + "".join(out) - + "" - ) + return _document(f"Benchmark — {e(results['skill'])}", "".join(out)) + + +_FRAGILE_LABEL = {"efficacy": "narrow efficacy", "regression": "near regressing"} + + +def render_leaderboard(rows, generated_at=""): + """The all-skills index: one linked row per skill carrying both verdicts.""" + e = html.escape + out = ["
", "

Skill benchmarks

"] + meta = [e(generated_at)] if generated_at else [] + meta.append(_plural(len(rows), "skill")) + out.append(f"
{' · '.join(meta)}
") + + out.append("" + "" + "") + for r in rows: + chips = "".join( + f"{_FRAGILE_LABEL[axis]}" + for axis in r["fragileAxes"] + ) + out.append( + f"" + f"" + f"" + ) + out.append("
SkillEfficacyRegression
{e(r['skill'])}{chips}{_state_badge(r['efficacyVerdict'])}{_state_badge(r['regressionVerdict'])}
") + + return _document("Skill benchmarks", "".join(out)) + + +def _write_outputs(model, render, json_out, html_out): + """Emit a model as JSON and/or rendered HTML, or to stdout if neither given.""" + if json_out: + with open(json_out, "w") as f: + json.dump(model, f, indent=2) + if html_out: + with open(html_out, "w") as f: + f.write(render()) + if not json_out and not html_out: + json.dump(model, sys.stdout, indent=2) + sys.stdout.write("\n") def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("bundle", help="path to the run-bundle JSON") - parser.add_argument("--json", dest="json_out", help="write the results model here") - parser.add_argument("--html", dest="html_out", help="write the HTML report here") + parser.add_argument( + "inputs", + nargs="+", + help="the run-bundle JSON, or the per-skill results JSONs with --leaderboard", + ) + parser.add_argument("--json", dest="json_out", help="write the model here") + parser.add_argument("--html", dest="html_out", help="write the HTML here") parser.add_argument( "--history", dest="history", help="append this run to the JSON-lines history file and draw the trend ribbons", ) + parser.add_argument( + "--leaderboard", + action="store_true", + help="render the index leaderboard over the given per-skill results JSONs", + ) + parser.add_argument( + "--generated-at", + dest="generated_at", + default="", + help="timestamp stamped on the leaderboard index", + ) args = parser.parse_args(argv) - with open(args.bundle) as f: + if args.leaderboard: + if args.history: + parser.error("--history does not apply in leaderboard mode") + models = [] + for path in args.inputs: + with open(path) as f: + models.append(json.load(f)) + rows = build_leaderboard(models) + _write_outputs( + rows, lambda: render_leaderboard(rows, args.generated_at), + args.json_out, args.html_out, + ) + return 0 + + if len(args.inputs) != 1: + parser.error("scoring a run takes exactly one bundle; use --leaderboard for many") + + with open(args.inputs[0]) as f: bundle = json.load(f) results = build_results(bundle) @@ -684,15 +841,9 @@ def main(argv=None): write_history(args.history, trim_history(series)) results["trend"] = build_trend(series) - if args.json_out: - with open(args.json_out, "w") as f: - json.dump(results, f, indent=2) - if args.html_out: - with open(args.html_out, "w") as f: - f.write(render_html(results)) - if not args.json_out and not args.html_out: - json.dump(results, sys.stdout, indent=2) - sys.stdout.write("\n") + _write_outputs( + results, lambda: render_html(results), args.json_out, args.html_out + ) return 0