alexion
97c0eeb055
Add a second pure transform to the benchmark core: a `--leaderboard` mode that ranks per-skill results models into an index leaderboard, one row per skill carrying both verdicts, links out to each per-skill report, and sorts any red first (regressions above efficacy failures), then fragile-but-passing, then clean green. The fragile tier reuses the 0006 per-badge chips: the efficacy chip now carries its win count so the leaderboard reads it against the pass floor without recomputing margins, and fragility is scoped to the passing tier so a red row never carries a chip. The runner's SKILL.md gains the no-argument batch flow and the index invocation. The fixture test covers the tiered sort, the not-applicable Regression cell, and the per-skill links.
853 lines
33 KiB
Python
853 lines
33 KiB
Python
#!/usr/bin/env python3
|
|
"""Deterministic scoring and rendering for a skill benchmark run.
|
|
|
|
A pure transform: given a collected run bundle (per-arm transcript usage,
|
|
hard-assertion results, and the per-trial judge verdicts for each comparison) it
|
|
produces the results model and a self-contained HTML report. No LLM, no network,
|
|
no clock — the `generatedAt` stamp is supplied by the caller so the transform
|
|
stays pure.
|
|
|
|
The model is parameterized over the arms and the comparisons, so one path serves
|
|
both the three-arm run (efficacy against no-skill and regression against the
|
|
previous version) and the degenerate two-arm run with no previous version.
|
|
|
|
When `--history` names a per-skill history file, the run's summary line is
|
|
appended to it, the file is trimmed oldest-first, and the report grows two trend
|
|
ribbons plotted from the recent tail. That file read-and-rewrite is the only side
|
|
effect, opt-in and deterministic given the same file contents and inputs.
|
|
|
|
Given `--leaderboard`, the positional arguments are read as several per-skill
|
|
results models instead, and the transform yields the index leaderboard: one row
|
|
per skill carrying both verdicts, sorted with regressions first, then efficacy
|
|
failures, then fragile-but-passing skills, then clean green.
|
|
|
|
Usage:
|
|
benchmark_core.py <bundle.json> [--json <out.json>] [--html <out.html>]
|
|
[--history <hist.jsonl>]
|
|
benchmark_core.py --leaderboard <results.json>... [--json <out.json>]
|
|
[--html <out.html>] [--generated-at <stamp>]
|
|
|
|
With no output flags it writes the model as JSON to stdout.
|
|
"""
|
|
|
|
import argparse
|
|
import html
|
|
import json
|
|
import sys
|
|
|
|
# Opus 4.8 per-million-token rates, in US dollars.
|
|
# Cache creation is the 5-minute-TTL write, priced at 1.25x input.
|
|
# Cache read is priced at 0.1x input.
|
|
# Cost is reported alongside quality but never gates a verdict.
|
|
DEFAULT_PRICING = {
|
|
"input": 5.0,
|
|
"output": 25.0,
|
|
"cacheCreation": 6.25,
|
|
"cacheRead": 0.5,
|
|
}
|
|
|
|
USAGE_COMPONENTS = ("input", "output", "cacheCreation", "cacheRead")
|
|
|
|
WIN, TIE, LOSS = "WIN", "TIE", "LOSS"
|
|
|
|
# The per-skill history is a JSON-lines tail of run summaries kept in the
|
|
# git-ignored reports directory.
|
|
# It is capped and trimmed oldest-first, and the ribbons plot only its most
|
|
# recent window.
|
|
HISTORY_CAP = 50
|
|
RIBBON_WINDOW = 7
|
|
|
|
# Efficacy passes at three wins, so this is the floor a green case sits above.
|
|
EFFICACY_WINS_FLOOR = 3
|
|
|
|
|
|
def _arm_usage_totals(bundle, arm_id):
|
|
"""Sum an arm's transcript usage across every case and trial."""
|
|
totals = {c: 0 for c in USAGE_COMPONENTS}
|
|
turns = 0
|
|
for case in bundle["cases"]:
|
|
for trial in case["trials"]:
|
|
usage = trial["usage"][arm_id]
|
|
for c in USAGE_COMPONENTS:
|
|
totals[c] += usage.get(c, 0)
|
|
turns += usage.get("turns", 0)
|
|
return totals, turns
|
|
|
|
|
|
def _arm_metrics(bundle, arm_id, pricing):
|
|
components, turns = _arm_usage_totals(bundle, arm_id)
|
|
raw_tokens = sum(components.values())
|
|
imputed_cost = sum(components[c] * pricing[c] for c in USAGE_COMPONENTS) / 1_000_000
|
|
# Cost-equivalent tokens: the dollar cost expressed in units of base input
|
|
# tokens, so a single figure captures the pricing-weighted total.
|
|
cost_equivalent_tokens = round(imputed_cost * 1_000_000 / pricing["input"])
|
|
return {
|
|
"turns": turns,
|
|
"rawTokens": raw_tokens,
|
|
"components": components,
|
|
"costEquivalentTokens": cost_equivalent_tokens,
|
|
"imputedCost": imputed_cost,
|
|
}
|
|
|
|
|
|
def _collapse_trial(verdict, comparison):
|
|
"""A trial's head-to-head becomes WIN, TIE, or LOSS for the new arm.
|
|
|
|
A tie is a non-win. Any winner that is neither the new nor the against arm
|
|
(a malformed verdict) is treated as a tie rather than crashing the run.
|
|
"""
|
|
winner = verdict.get("winner")
|
|
if winner == comparison["new"]:
|
|
return WIN
|
|
if winner == comparison["against"]:
|
|
return LOSS
|
|
return TIE
|
|
|
|
|
|
# Each comparison names a pass rule keyed here.
|
|
# Efficacy hunts for a reliable win, so it needs a wins floor.
|
|
# Regression hunts for a degradation, so a tie is already a success and only losses matter.
|
|
PASS_RULES = {
|
|
"efficacy": lambda wins, losses: wins >= 3 and losses <= 1,
|
|
"regression": lambda wins, losses: losses <= 1,
|
|
}
|
|
|
|
|
|
def _comparison_by_rule(comparisons, rule):
|
|
"""The comparison carrying a given pass rule, or None if absent."""
|
|
return next((c for c in comparisons if c["rule"] == rule), None)
|
|
|
|
|
|
def _score_case(case, comparisons):
|
|
hard_failed = any(
|
|
t.get("hard", {}).get("ran") and not t["hard"].get("pass", False)
|
|
for t in case["trials"]
|
|
)
|
|
|
|
per_comparison = {}
|
|
for comp in comparisons:
|
|
strip, flagged, evidence = [], [], []
|
|
for i, trial in enumerate(case["trials"]):
|
|
outcome = _collapse_trial(trial["comparisons"][comp["id"]], comp)
|
|
strip.append(outcome)
|
|
if outcome == LOSS:
|
|
flagged.append(i)
|
|
# The losing-trial output pair lets the report surface the new
|
|
# arm's output beside the one it lost to, for a human to eyeball.
|
|
outputs = trial.get("outputs", {})
|
|
evidence.append({
|
|
"trial": i,
|
|
"new": outputs.get(comp["new"], ""),
|
|
"against": outputs.get(comp["against"], ""),
|
|
})
|
|
wins = strip.count(WIN)
|
|
ties = strip.count(TIE)
|
|
losses = strip.count(LOSS)
|
|
head_to_head = PASS_RULES[comp["rule"]](wins, losses)
|
|
# The hard gate is new-arm-only and fails efficacy outright.
|
|
# It catches the current skill emitting malformed output.
|
|
# Regression is relative to the previous version, so it is judged purely on losses.
|
|
passed = head_to_head and not (hard_failed and comp["rule"] == "efficacy")
|
|
per_comparison[comp["id"]] = {
|
|
"trials": strip,
|
|
"wins": wins,
|
|
"ties": ties,
|
|
"losses": losses,
|
|
"passed": passed,
|
|
"flaggedLosses": flagged,
|
|
"evidence": evidence,
|
|
}
|
|
|
|
efficacy_id = _comparison_by_rule(comparisons, "efficacy")["id"]
|
|
regression = _comparison_by_rule(comparisons, "regression")
|
|
|
|
return {
|
|
"name": case["name"],
|
|
"description": case.get("description", ""),
|
|
"softCriteria": case.get("softCriteria", []),
|
|
"hardFailed": hard_failed,
|
|
"comparisons": per_comparison,
|
|
"efficacyPassed": per_comparison[efficacy_id]["passed"],
|
|
"regressionPassed": (
|
|
per_comparison[regression["id"]]["passed"] if regression else None
|
|
),
|
|
}
|
|
|
|
|
|
def _net_margin(cases, comparison_id):
|
|
"""Wins minus losses for a comparison, summed across every case."""
|
|
total = 0
|
|
for case in cases:
|
|
comp = case["comparisons"][comparison_id]
|
|
total += comp["wins"] - comp["losses"]
|
|
return total
|
|
|
|
|
|
def _flag_fragility(cases, efficacy_id, regression, efficacy_green, regression_green):
|
|
"""Chip the most fragile passing case on each green axis.
|
|
|
|
A green run still names its weakest case so a barely-green skill cannot pass
|
|
for robust. The efficacy chip lands on the single case closest to dropping
|
|
under the three-win floor, ties broken by authored order. The regression chip
|
|
lands on every case sitting at its one tolerated loss, so a case fragile on
|
|
both axes carries both.
|
|
"""
|
|
for case in cases:
|
|
case["chips"] = []
|
|
|
|
if efficacy_green and cases:
|
|
narrowest = min(cases, key=lambda c: c["comparisons"][efficacy_id]["wins"])
|
|
wins = narrowest["comparisons"][efficacy_id]["wins"]
|
|
narrowest["chips"].append({
|
|
"axis": "efficacy",
|
|
"wins": wins,
|
|
"label": f"narrowest efficacy margin · {wins}W",
|
|
})
|
|
|
|
if regression is not None and regression_green:
|
|
for case in cases:
|
|
if case["comparisons"][regression["id"]]["losses"] == 1:
|
|
case["chips"].append({
|
|
"axis": "regression",
|
|
"label": "one loss from regressing",
|
|
})
|
|
|
|
|
|
def build_results(bundle):
|
|
"""Turn a run bundle into the results model."""
|
|
pricing = DEFAULT_PRICING
|
|
comparisons = bundle["comparisons"]
|
|
arms = bundle["arms"]
|
|
|
|
regression = _comparison_by_rule(comparisons, "regression")
|
|
|
|
arm_metrics = {a["id"]: _arm_metrics(bundle, a["id"], pricing) for a in arms}
|
|
cases = [_score_case(c, comparisons) for c in bundle["cases"]]
|
|
|
|
efficacy_id = _comparison_by_rule(comparisons, "efficacy")["id"]
|
|
efficacy_green = all(c["efficacyPassed"] for c in cases)
|
|
# No previous-version arm means the regression axis has nothing to measure,
|
|
# so it reads not-applicable rather than green or red.
|
|
if regression is None:
|
|
regression_verdict = "not-applicable"
|
|
regression_net = None
|
|
else:
|
|
regression_verdict = (
|
|
"green" if all(c["regressionPassed"] for c in cases) else "red"
|
|
)
|
|
regression_net = _net_margin(cases, regression["id"])
|
|
|
|
_flag_fragility(
|
|
cases, efficacy_id, regression, efficacy_green, regression_verdict == "green"
|
|
)
|
|
|
|
return {
|
|
"skill": bundle["skill"],
|
|
"generatedAt": bundle.get("generatedAt", ""),
|
|
"temperature": bundle.get("temperature"),
|
|
"trialsPerCase": bundle.get("trialsPerCase"),
|
|
"armShape": "three-arm" if regression is not None else "two-arm",
|
|
"arms": arms,
|
|
"comparisons": comparisons,
|
|
"efficacyVerdict": "green" if efficacy_green else "red",
|
|
"regressionVerdict": regression_verdict,
|
|
"efficacyNetMargin": _net_margin(cases, efficacy_id),
|
|
"regressionNetMargin": regression_net,
|
|
"armMetrics": arm_metrics,
|
|
"cases": cases,
|
|
}
|
|
|
|
|
|
# --- History and trend --------------------------------------------------------
|
|
|
|
|
|
def summary_line(results):
|
|
"""The one-line run summary appended to the per-skill history.
|
|
|
|
It records each axis's net margin and pass/fail plus the arm shape — all the
|
|
ribbons need to redraw both sparklines and show a gap for any two-arm run. A
|
|
two-arm run has no regression measurement, so both its regression fields are
|
|
null.
|
|
"""
|
|
three_arm = results["armShape"] == "three-arm"
|
|
return {
|
|
"generatedAt": results.get("generatedAt", ""),
|
|
"armShape": results["armShape"],
|
|
"efficacyNet": results["efficacyNetMargin"],
|
|
"efficacyPass": results["efficacyVerdict"] == "green",
|
|
"regressionNet": results["regressionNetMargin"],
|
|
"regressionPass": (results["regressionVerdict"] == "green") if three_arm else None,
|
|
}
|
|
|
|
|
|
def load_history(path):
|
|
"""Read the JSON-lines history, tolerating a missing or empty file."""
|
|
try:
|
|
with open(path) as f:
|
|
return [json.loads(line) for line in f if line.strip()]
|
|
except FileNotFoundError:
|
|
return []
|
|
|
|
|
|
def trim_history(lines):
|
|
"""Keep the most recent runs, dropping the oldest past the cap."""
|
|
return lines[-HISTORY_CAP:]
|
|
|
|
|
|
def write_history(path, lines):
|
|
"""Rewrite the history file with the given lines, oldest first."""
|
|
with open(path, "w") as f:
|
|
for line in lines:
|
|
f.write(json.dumps(line) + "\n")
|
|
|
|
|
|
def build_trend(series):
|
|
"""Ribbon data for both axes over the most recent window of runs.
|
|
|
|
A run with no value on an axis — a two-arm run on the regression axis — plots
|
|
no dot and breaks the line, so the gap reads as absent rather than being
|
|
interpolated across. The delta compares against the previous run that actually
|
|
carries a value on that axis, so a two-arm gap does not blank the comparison.
|
|
"""
|
|
window = series[-RIBBON_WINDOW:]
|
|
|
|
def axis(net_key, pass_key):
|
|
points = [
|
|
{
|
|
"net": run.get(net_key),
|
|
"pass": run.get(pass_key),
|
|
"current": i == len(window) - 1,
|
|
}
|
|
for i, run in enumerate(window)
|
|
]
|
|
valued = [p for p in points if p["net"] is not None]
|
|
current_net = points[-1]["net"] if points else None
|
|
delta = None
|
|
if current_net is not None and len(valued) >= 2:
|
|
delta = valued[-1]["net"] - valued[-2]["net"]
|
|
return {
|
|
"points": points,
|
|
"current": current_net,
|
|
"delta": delta,
|
|
"green": sum(1 for p in points if p["pass"] is True),
|
|
"applicable": len(valued),
|
|
}
|
|
|
|
return {
|
|
"efficacy": axis("efficacyNet", "efficacyPass"),
|
|
"regression": axis("regressionNet", "regressionPass"),
|
|
}
|
|
|
|
|
|
# --- All-skills leaderboard ---------------------------------------------------
|
|
|
|
|
|
def _fragile_axes(results):
|
|
"""The axes on which a passing skill sits one trial-flip from failing.
|
|
|
|
Reuses the per-badge fragility chips rather than recomputing margins: the
|
|
regression chip already marks a case one loss from regressing, and the
|
|
efficacy chip marks the narrowest case, fragile exactly when its win count
|
|
rests on the pass floor. A green run always chips its narrowest efficacy
|
|
case, so the floor test is what tells a barely-green skill from a roomy one.
|
|
"""
|
|
axes = set()
|
|
for case in results["cases"]:
|
|
for chip in case.get("chips", []):
|
|
if chip["axis"] == "regression":
|
|
axes.add("regression")
|
|
elif chip["axis"] == "efficacy" and chip.get("wins") == EFFICACY_WINS_FLOOR:
|
|
axes.add("efficacy")
|
|
return sorted(axes)
|
|
|
|
|
|
def _leaderboard_tier(results, fragile):
|
|
"""The sort tier, most urgent first.
|
|
|
|
A regression outranks an efficacy failure because it means a skill that was
|
|
working just broke — the sharper signal mid-edit. A passing skill still one
|
|
flip from failing sorts above a clean sweep.
|
|
"""
|
|
if results["regressionVerdict"] == "red":
|
|
return 0
|
|
if results["efficacyVerdict"] == "red":
|
|
return 1
|
|
if fragile:
|
|
return 2
|
|
return 3
|
|
|
|
|
|
def build_leaderboard(models):
|
|
"""Rank per-skill results into leaderboard rows, most urgent first.
|
|
|
|
Rows within a tier order by skill name, so the board diffs cleanly run to run.
|
|
"""
|
|
rows = []
|
|
for m in models:
|
|
axes = _fragile_axes(m)
|
|
tier = _leaderboard_tier(m, bool(axes))
|
|
rows.append({
|
|
"skill": m["skill"],
|
|
"efficacyVerdict": m["efficacyVerdict"],
|
|
"regressionVerdict": m["regressionVerdict"],
|
|
# Fragility marks a passing skill one flip from failing, so a red skill
|
|
# carries no chip even when a still-passing axis is at its edge.
|
|
"fragileAxes": axes if tier == 2 else [],
|
|
"tier": tier,
|
|
"href": f"{m['skill']}.html",
|
|
})
|
|
rows.sort(key=lambda r: (r["tier"], r["skill"]))
|
|
return rows
|
|
|
|
|
|
# --- HTML rendering -----------------------------------------------------------
|
|
|
|
_STYLE = """
|
|
:root { color-scheme: light; }
|
|
body { font: 15px/1.5 -apple-system, system-ui, sans-serif; margin: 0; padding: 2rem;
|
|
color: #1a1a1a; background: #fafafa; }
|
|
.wrap { max-width: 60rem; margin: 0 auto; }
|
|
h1 { font-size: 1.4rem; margin: 0 0 .25rem; }
|
|
.badges { display: flex; gap: .5rem; margin: .25rem 0; flex-wrap: wrap; }
|
|
.badge { display: inline-block; padding: .2rem .7rem; border-radius: 999px;
|
|
font-weight: 600; font-size: .85rem; }
|
|
.badge.green { background: #d6f5df; color: #0f6b34; }
|
|
.badge.red { background: #fbdcdc; color: #9b1c1c; }
|
|
.badge.na { background: #eee; color: #666; }
|
|
.headline { font-size: 1.02rem; font-weight: 600; margin: .8rem 0 .2rem; }
|
|
.meta { color: #666; font-size: .85rem; margin: .5rem 0 1rem; }
|
|
.ribbons { display: grid; gap: .5rem; margin: .5rem 0 1.5rem; }
|
|
.ribbon { display: grid; grid-template-columns: 6.5rem 1fr auto; align-items: center;
|
|
gap: 1rem; border: 1px solid #e5e5e5; border-radius: 8px;
|
|
padding: .5rem .9rem; background: #fff; }
|
|
.rlabel { font-weight: 600; font-size: .85rem; }
|
|
.spark { width: 100%; height: 44px; display: block; }
|
|
.spark .line { fill: none; stroke: #c4c4c4; stroke-width: 1.5; }
|
|
.spark .zero { stroke: #e5e5e5; stroke-width: 1; stroke-dasharray: 2 3; }
|
|
.spark .dot.pass { fill: #2f9e5f; }
|
|
.spark .dot.fail { fill: #c0392b; }
|
|
.spark .ring { fill: none; stroke: #1a1a1a; stroke-width: 1.5; }
|
|
.readout { font-size: .8rem; color: #555; white-space: nowrap; text-align: right; }
|
|
.readout .up { color: #0f6b34; font-weight: 600; }
|
|
.readout .down { color: #9b1c1c; font-weight: 600; }
|
|
.chip { display: inline-block; margin-left: .5rem; padding: .05rem .55rem;
|
|
border-radius: 999px; font-size: .7rem; font-weight: 600;
|
|
vertical-align: middle; }
|
|
.chip.efficacy { background: #fff1cf; color: #8a5a00; }
|
|
.chip.regression { background: #e9e2fb; color: #5b3ea8; }
|
|
table { border-collapse: collapse; width: 100%; margin: .5rem 0; font-size: .9rem; }
|
|
th, td { text-align: right; padding: .4rem .6rem; border-bottom: 1px solid #e5e5e5; }
|
|
th:first-child, td:first-child { text-align: left; }
|
|
thead th { border-bottom: 2px solid #ccc; }
|
|
.footnote { color: #777; font-size: .8rem; margin: .3rem 0 1.5rem; }
|
|
.board td, .board th { text-align: left; }
|
|
.board td.verdict, .board th.verdict { text-align: right; }
|
|
.board a { color: #1a56c4; text-decoration: none; font-weight: 600; }
|
|
.board a:hover { text-decoration: underline; }
|
|
.case { border: 1px solid #e5e5e5; border-radius: 8px; padding: 1rem 1.25rem;
|
|
margin: .75rem 0; background: #fff; }
|
|
.case.fail { border-color: #f0b6b6; }
|
|
.case .desc { color: #666; font-size: .85rem; margin: 0 0 .6rem; }
|
|
.strip { display: flex; gap: .3rem; margin: .3rem 0; flex-wrap: wrap; }
|
|
.cell { width: 2.6rem; text-align: center; padding: .2rem 0; border-radius: 4px;
|
|
font-size: .72rem; font-weight: 700; letter-spacing: .02em; }
|
|
.cell.WIN { background: #d6f5df; color: #0f6b34; }
|
|
.cell.TIE { background: #eee; color: #555; }
|
|
.cell.LOSS { background: #fbdcdc; color: #9b1c1c; }
|
|
.result { font-size: .85rem; font-weight: 600; }
|
|
.result.pass { color: #0f6b34; }
|
|
.result.fail { color: #9b1c1c; }
|
|
.tag { font-size: .75rem; color: #9b1c1c; font-weight: 600; }
|
|
.crit { color: #555; font-size: .82rem; margin: .5rem 0 0; padding-left: 1.1rem; }
|
|
.case > summary { cursor: pointer; font-weight: 600; font-size: 1rem;
|
|
list-style-position: outside; }
|
|
.case > summary::marker { color: #999; }
|
|
.cmps { display: grid; grid-template-columns: repeat(auto-fit, minmax(18rem, 1fr));
|
|
gap: 1rem; align-items: start; margin: .5rem 0 .2rem; }
|
|
.cmp { min-width: 0; }
|
|
.evidence { display: grid; grid-template-columns: repeat(auto-fit, minmax(12rem, 1fr));
|
|
gap: .6rem; margin: .4rem 0 0; }
|
|
.evidence figure { margin: 0; }
|
|
.evidence figcaption { font-size: .72rem; color: #666; font-weight: 600;
|
|
margin: 0 0 .2rem; }
|
|
.evidence pre { margin: 0; padding: .5rem .6rem; background: #f6f6f6;
|
|
border: 1px solid #e5e5e5; border-radius: 4px; font-size: .78rem;
|
|
white-space: pre-wrap; overflow-wrap: anywhere; }
|
|
.evidence .lead { grid-column: 1 / -1; font-size: .78rem; color: #9b1c1c;
|
|
font-weight: 600; margin: .3rem 0 0; }
|
|
"""
|
|
|
|
|
|
def _document(title, body):
|
|
"""Wrap rendered body markup in the self-contained HTML document shell."""
|
|
return (
|
|
"<!doctype html><html><head><meta charset=\"utf-8\">"
|
|
f"<title>{title}</title><style>{_STYLE}</style></head><body>"
|
|
+ body
|
|
+ "</body></html>"
|
|
)
|
|
|
|
|
|
def _fmt_int(n):
|
|
return f"{n:,}"
|
|
|
|
|
|
def _fmt_cost(d):
|
|
return f"${d:,.4f}"
|
|
|
|
|
|
def _plural(n, noun):
|
|
return f"{n} {noun}" if n == 1 else f"{n} {noun}s"
|
|
|
|
|
|
def _headline(results):
|
|
"""One 'so what' sentence reading the two verdicts together.
|
|
|
|
Every clause it states is backed by a verdict or a case count rendered on the
|
|
same screen.
|
|
"""
|
|
eff = results["efficacyVerdict"]
|
|
reg = results["regressionVerdict"]
|
|
eff_fail = sum(1 for c in results["cases"] if not c["efficacyPassed"])
|
|
reg_fail = sum(1 for c in results["cases"] if c["regressionPassed"] is False)
|
|
|
|
if reg == "not-applicable":
|
|
if eff == "green":
|
|
return "This skill earns its keep against no-skill, with no previous version to regress against."
|
|
return (f"This skill does not beat no-skill on {_plural(eff_fail, 'case')}, "
|
|
"with no previous version to regress against.")
|
|
if eff == "green" and reg == "green":
|
|
return "This skill earns its keep and this edit held quality steady."
|
|
if eff == "green" and reg == "red":
|
|
return f"Still valuable, but this edit regressed {_plural(reg_fail, 'case')}."
|
|
if eff == "red" and reg == "green":
|
|
return (f"This skill does not beat no-skill on {_plural(eff_fail, 'case')}, "
|
|
"but this edit did not make it worse.")
|
|
return (f"This skill does not beat no-skill on {_plural(eff_fail, 'case')} "
|
|
f"and this edit regressed {_plural(reg_fail, 'case')}.")
|
|
|
|
|
|
_BADGE_TEXT = {"green": "GREEN", "red": "RED", "not-applicable": "N/A"}
|
|
_BADGE_CLASS = {"green": "green", "red": "red", "not-applicable": "na"}
|
|
|
|
|
|
def _state_badge(state):
|
|
return f"<span class=\"badge {_BADGE_CLASS[state]}\">{_BADGE_TEXT[state]}</span>"
|
|
|
|
|
|
def _badge(label, state):
|
|
return f"<span class=\"badge {_BADGE_CLASS[state]}\">{label}: {_BADGE_TEXT[state]}</span>"
|
|
|
|
|
|
def _sparkline(points, width=240, height=44):
|
|
"""An inline SVG net-margin sparkline where gaps break the line and drop no dot."""
|
|
nets = [p["net"] for p in points if p["net"] is not None]
|
|
if not nets:
|
|
return f"<svg viewBox=\"0 0 {width} {height}\" class=\"spark\"></svg>"
|
|
|
|
pad_x, pad_y = 10, 9
|
|
lo, hi = min(nets), max(nets)
|
|
n = len(points)
|
|
|
|
def x(i):
|
|
return width / 2 if n == 1 else pad_x + i * (width - 2 * pad_x) / (n - 1)
|
|
|
|
def y(v):
|
|
if hi == lo:
|
|
return height / 2
|
|
return height - pad_y - (v - lo) / (hi - lo) * (height - 2 * pad_y)
|
|
|
|
parts = [
|
|
f"<svg viewBox=\"0 0 {width} {height}\" class=\"spark\" "
|
|
"preserveAspectRatio=\"xMidYMid meet\" xmlns=\"http://www.w3.org/2000/svg\">"
|
|
]
|
|
if lo < 0 < hi:
|
|
zy = y(0)
|
|
parts.append(f"<line x1=\"0\" y1=\"{zy:.1f}\" x2=\"{width}\" y2=\"{zy:.1f}\" class=\"zero\"/>")
|
|
|
|
# Split the run into contiguous valued segments so a gap leaves a real break.
|
|
segments, current = [], []
|
|
for i, p in enumerate(points):
|
|
if p["net"] is None:
|
|
if current:
|
|
segments.append(current)
|
|
current = []
|
|
else:
|
|
current.append((x(i), y(p["net"])))
|
|
if current:
|
|
segments.append(current)
|
|
for seg in segments:
|
|
if len(seg) >= 2:
|
|
coords = " ".join(f"{px:.1f},{py:.1f}" for px, py in seg)
|
|
parts.append(f"<polyline points=\"{coords}\" class=\"line\"/>")
|
|
|
|
for i, p in enumerate(points):
|
|
if p["net"] is None:
|
|
continue
|
|
cx, cy = x(i), y(p["net"])
|
|
if p["current"]:
|
|
parts.append(f"<circle cx=\"{cx:.1f}\" cy=\"{cy:.1f}\" r=\"5.5\" class=\"ring\"/>")
|
|
cls = "dot pass" if p["pass"] else "dot fail"
|
|
parts.append(f"<circle cx=\"{cx:.1f}\" cy=\"{cy:.1f}\" r=\"3\" class=\"{cls}\"/>")
|
|
|
|
parts.append("</svg>")
|
|
return "".join(parts)
|
|
|
|
|
|
def _ribbon_readout(axis):
|
|
"""The net / delta-vs-previous / green-count line beneath a ribbon."""
|
|
net = "net —" if axis["current"] is None else f"net {axis['current']:+d}"
|
|
|
|
delta = axis["delta"]
|
|
if delta is None:
|
|
change = ""
|
|
elif delta > 0:
|
|
change = f"<span class=\"up\">▲ {delta}</span> vs prev"
|
|
elif delta < 0:
|
|
change = f"<span class=\"down\">▼ {abs(delta)}</span> vs prev"
|
|
else:
|
|
change = "±0 vs prev"
|
|
|
|
if axis["applicable"]:
|
|
green = f"{axis['green']}/{axis['applicable']} green"
|
|
else:
|
|
green = "no runs yet"
|
|
|
|
return " · ".join(bit for bit in (net, change, green) if bit)
|
|
|
|
|
|
def _render_ribbons(results):
|
|
"""Two stacked net-margin ribbons, empty when no history was supplied."""
|
|
trend = results.get("trend")
|
|
if not trend:
|
|
return ""
|
|
out = ["<div class=\"ribbons\">"]
|
|
for key, label in (("efficacy", "Efficacy"), ("regression", "Regression")):
|
|
axis = trend[key]
|
|
out.append("<div class=\"ribbon\">")
|
|
out.append(f"<div class=\"rlabel\">{label}</div>")
|
|
out.append(_sparkline(axis["points"]))
|
|
out.append(f"<div class=\"readout\">{_ribbon_readout(axis)}</div>")
|
|
out.append("</div>")
|
|
out.append("</div>")
|
|
return "".join(out)
|
|
|
|
|
|
def render_html(results):
|
|
e = html.escape
|
|
arm_labels = {a["id"]: a["label"] for a in results["arms"]}
|
|
three_arm = results["armShape"] == "three-arm"
|
|
out = []
|
|
out.append("<div class=\"wrap\">")
|
|
out.append(f"<h1>Benchmark — {e(results['skill'])}</h1>")
|
|
out.append("<div class=\"badges\">")
|
|
out.append(_badge("Efficacy", results["efficacyVerdict"]))
|
|
out.append(_badge("Regression", results["regressionVerdict"]))
|
|
out.append("</div>")
|
|
|
|
meta_bits = [f"arm shape: {e(results['armShape'])}"]
|
|
if results.get("trialsPerCase") is not None:
|
|
meta_bits.append(f"{results['trialsPerCase']} trials/case")
|
|
if results.get("temperature") is not None:
|
|
meta_bits.append(f"temperature {results['temperature']}")
|
|
if results.get("generatedAt"):
|
|
meta_bits.append(e(results["generatedAt"]))
|
|
out.append(f"<div class=\"meta\">{' · '.join(meta_bits)}</div>")
|
|
|
|
out.append(_render_ribbons(results))
|
|
|
|
out.append(f"<p class=\"headline\">{e(_headline(results))}</p>")
|
|
|
|
# Per-arm cost table.
|
|
out.append("<table><thead><tr><th>Arm</th><th>Turns</th><th>Raw tokens</th>"
|
|
"<th>Cost-equiv tokens</th><th>Imputed cost</th></tr></thead><tbody>")
|
|
for arm in results["arms"]:
|
|
m = results["armMetrics"][arm["id"]]
|
|
out.append(
|
|
f"<tr><td>{e(arm['label'])}</td><td>{_fmt_int(m['turns'])}</td>"
|
|
f"<td>{_fmt_int(m['rawTokens'])}</td>"
|
|
f"<td>{_fmt_int(m['costEquivalentTokens'])}</td>"
|
|
f"<td>{_fmt_cost(m['imputedCost'])}</td></tr>"
|
|
)
|
|
out.append("</tbody></table>")
|
|
if three_arm:
|
|
footnote = ("Absolute cost is inflated by shared-context cache overhead, "
|
|
"so the trustworthy signals are two ratios: new-vs-no-skill "
|
|
"(what the skill costs over nothing) and new-vs-old (what this "
|
|
"edit added or saved), not the absolute figures.")
|
|
else:
|
|
footnote = ("Absolute cost is inflated by shared-context cache overhead, "
|
|
"so the trustworthy signal is the new-vs-no-skill ratio, not "
|
|
"the absolute figures.")
|
|
out.append(f"<p class=\"footnote\">{footnote}</p>")
|
|
|
|
# Cases in stable authored order, each a collapsible panel that auto-expands
|
|
# when either comparison fails or carries a flagged loss.
|
|
for case in results["cases"]:
|
|
comps = case["comparisons"]
|
|
expanded = case["hardFailed"] or any(
|
|
c["flaggedLosses"] or not c["passed"] for c in comps.values()
|
|
)
|
|
failed = (not case["efficacyPassed"]) or case["regressionPassed"] is False
|
|
cls = "case fail" if failed else "case"
|
|
opened = " open" if expanded else ""
|
|
chips = "".join(
|
|
f"<span class=\"chip {ch['axis']}\">{e(ch['label'])}</span>"
|
|
for ch in case.get("chips", [])
|
|
)
|
|
out.append(f"<details class=\"{cls}\"{opened}>")
|
|
out.append(f"<summary>{e(case['name'])}{chips}</summary>")
|
|
if case["description"]:
|
|
out.append(f"<p class=\"desc\">{e(case['description'])}</p>")
|
|
if case["hardFailed"]:
|
|
out.append("<p class=\"tag\">Hard assertion failed — case fails outright.</p>")
|
|
out.append("<div class=\"cmps\">")
|
|
for comp in results["comparisons"]:
|
|
c = comps[comp["id"]]
|
|
out.append("<div class=\"cmp\">")
|
|
out.append(f"<div><strong>{e(comp['label'])}</strong></div>")
|
|
out.append("<div class=\"strip\">")
|
|
for outcome in c["trials"]:
|
|
out.append(f"<span class=\"cell {outcome}\">{outcome}</span>")
|
|
out.append("</div>")
|
|
rc = "pass" if c["passed"] else "fail"
|
|
summary = (f"{c['wins']}W / {c['ties']}T / {c['losses']}L — "
|
|
f"{'PASS' if c['passed'] else 'FAIL'}")
|
|
out.append(f"<div class=\"result {rc}\">{summary}</div>")
|
|
if c["flaggedLosses"]:
|
|
trials = ", ".join(f"#{i + 1}" for i in c["flaggedLosses"])
|
|
out.append(f"<div class=\"tag\">Loss flagged for review: trial {trials}</div>")
|
|
new_label = arm_labels.get(comp["new"], comp["new"])
|
|
against_label = arm_labels.get(comp["against"], comp["against"])
|
|
for ev in c["evidence"]:
|
|
out.append("<div class=\"evidence\">")
|
|
out.append(f"<p class=\"lead\">Trial #{ev['trial'] + 1} — losing output pair</p>")
|
|
out.append(
|
|
f"<figure><figcaption>{e(new_label)}</figcaption>"
|
|
f"<pre>{e(ev['new'])}</pre></figure>"
|
|
)
|
|
out.append(
|
|
f"<figure><figcaption>{e(against_label)}</figcaption>"
|
|
f"<pre>{e(ev['against'])}</pre></figure>"
|
|
)
|
|
out.append("</div>")
|
|
out.append("</div>")
|
|
out.append("</div>")
|
|
if case["softCriteria"]:
|
|
out.append("<ul class=\"crit\">")
|
|
for crit in case["softCriteria"]:
|
|
out.append(f"<li>{e(crit)}</li>")
|
|
out.append("</ul>")
|
|
out.append("</details>")
|
|
|
|
out.append("</div>")
|
|
|
|
return _document(f"Benchmark — {e(results['skill'])}", "".join(out))
|
|
|
|
|
|
_FRAGILE_LABEL = {"efficacy": "narrow efficacy", "regression": "near regressing"}
|
|
|
|
|
|
def render_leaderboard(rows, generated_at=""):
|
|
"""The all-skills index: one linked row per skill carrying both verdicts."""
|
|
e = html.escape
|
|
out = ["<div class=\"wrap\">", "<h1>Skill benchmarks</h1>"]
|
|
meta = [e(generated_at)] if generated_at else []
|
|
meta.append(_plural(len(rows), "skill"))
|
|
out.append(f"<div class=\"meta\">{' · '.join(meta)}</div>")
|
|
|
|
out.append("<table class=\"board\"><thead><tr><th>Skill</th>"
|
|
"<th class=\"verdict\">Efficacy</th>"
|
|
"<th class=\"verdict\">Regression</th></tr></thead><tbody>")
|
|
for r in rows:
|
|
chips = "".join(
|
|
f"<span class=\"chip {axis}\">{_FRAGILE_LABEL[axis]}</span>"
|
|
for axis in r["fragileAxes"]
|
|
)
|
|
out.append(
|
|
f"<tr><td><a href=\"{e(r['href'])}\">{e(r['skill'])}</a>{chips}</td>"
|
|
f"<td class=\"verdict\">{_state_badge(r['efficacyVerdict'])}</td>"
|
|
f"<td class=\"verdict\">{_state_badge(r['regressionVerdict'])}</td></tr>"
|
|
)
|
|
out.append("</tbody></table></div>")
|
|
|
|
return _document("Skill benchmarks", "".join(out))
|
|
|
|
|
|
def _write_outputs(model, render, json_out, html_out):
|
|
"""Emit a model as JSON and/or rendered HTML, or to stdout if neither given."""
|
|
if json_out:
|
|
with open(json_out, "w") as f:
|
|
json.dump(model, f, indent=2)
|
|
if html_out:
|
|
with open(html_out, "w") as f:
|
|
f.write(render())
|
|
if not json_out and not html_out:
|
|
json.dump(model, sys.stdout, indent=2)
|
|
sys.stdout.write("\n")
|
|
|
|
|
|
def main(argv=None):
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument(
|
|
"inputs",
|
|
nargs="+",
|
|
help="the run-bundle JSON, or the per-skill results JSONs with --leaderboard",
|
|
)
|
|
parser.add_argument("--json", dest="json_out", help="write the model here")
|
|
parser.add_argument("--html", dest="html_out", help="write the HTML here")
|
|
parser.add_argument(
|
|
"--history",
|
|
dest="history",
|
|
help="append this run to the JSON-lines history file and draw the trend ribbons",
|
|
)
|
|
parser.add_argument(
|
|
"--leaderboard",
|
|
action="store_true",
|
|
help="render the index leaderboard over the given per-skill results JSONs",
|
|
)
|
|
parser.add_argument(
|
|
"--generated-at",
|
|
dest="generated_at",
|
|
default="",
|
|
help="timestamp stamped on the leaderboard index",
|
|
)
|
|
args = parser.parse_args(argv)
|
|
|
|
if args.leaderboard:
|
|
if args.history:
|
|
parser.error("--history does not apply in leaderboard mode")
|
|
models = []
|
|
for path in args.inputs:
|
|
with open(path) as f:
|
|
models.append(json.load(f))
|
|
rows = build_leaderboard(models)
|
|
_write_outputs(
|
|
rows, lambda: render_leaderboard(rows, args.generated_at),
|
|
args.json_out, args.html_out,
|
|
)
|
|
return 0
|
|
|
|
if len(args.inputs) != 1:
|
|
parser.error("scoring a run takes exactly one bundle; use --leaderboard for many")
|
|
|
|
with open(args.inputs[0]) as f:
|
|
bundle = json.load(f)
|
|
|
|
results = build_results(bundle)
|
|
|
|
if args.history:
|
|
series = load_history(args.history) + [summary_line(results)]
|
|
write_history(args.history, trim_history(series))
|
|
results["trend"] = build_trend(series)
|
|
|
|
_write_outputs(
|
|
results, lambda: render_html(results), args.json_out, args.html_out
|
|
)
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|