feat: add efficacy-benchmark tracer bullet (task 0004)
Introduce the thinnest complete path that benchmarks a skill's efficacy against a no-skill baseline and renders an HTML report. - tests/ convention: a top-level tree mirroring skills/ by full path, with case directories holding a case.md (CASE-FORMAT.md) and an optional fixture. - benchmark-skill: a distributed, non-model-invocable runner (/benchmark-skill <name>) that orchestrates force-invoked new-skill and skill-absent baseline arms plus a blind per-trial judge as in-session subagents. - Deterministic core (Python): a pure transform over the collected run data that sums per-arm usage, collapses each trial to WIN/TIE/LOSS, applies the efficacy pass rule (wins >= 3, losses <= 1), and renders a self-contained report. Parameterized over arms and comparisons; two-arm here. - Fixture-driven no-LLM check (checks/benchmark-core.nix) wired into flake.nix. - Real tests tree for axi-review; a live run produced its efficacy report.
This commit is contained in:
455
checks/fixtures/benchmark/run-bundle.json
Normal file
455
checks/fixtures/benchmark/run-bundle.json
Normal file
@@ -0,0 +1,455 @@
|
||||
{
|
||||
"skill": "sample-skill",
|
||||
"generatedAt": "2026-07-24T12:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
"arms": [
|
||||
{
|
||||
"id": "new",
|
||||
"label": "New skill"
|
||||
},
|
||||
{
|
||||
"id": "baseline",
|
||||
"label": "No skill"
|
||||
}
|
||||
],
|
||||
"comparisons": [
|
||||
{
|
||||
"id": "efficacy",
|
||||
"label": "Efficacy",
|
||||
"new": "new",
|
||||
"against": "baseline",
|
||||
"rule": "efficacy"
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "clean-pass",
|
||||
"description": "New skill reliably beats the no-skill baseline.",
|
||||
"softCriteria": [
|
||||
"The answer is grounded in the fixture.",
|
||||
"The answer follows the skill's format."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge preferred tie"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "regresses-baseline",
|
||||
"description": "New skill does not reliably beat the baseline.",
|
||||
"softCriteria": [
|
||||
"The answer resolves the user's request."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge preferred tie"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge preferred baseline"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge preferred baseline"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hard-gate-fail",
|
||||
"description": "Output wins the head-to-head but violates a hard assertion.",
|
||||
"softCriteria": [
|
||||
"The output is correct."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
}
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user