alexion
f4df5d31c9
Add the longitudinal layer to the benchmark core: a per-skill JSON-lines history that appends one summary line per run and trims oldest-first at fifty, two stacked net-margin trend ribbons (Efficacy and Regression, the latter leaving a gap for two-arm runs) with the current run ringed and a net/delta/green-count readout, and per-badge fragility chips that flag the narrowest passing case on each green axis so a barely-green skill cannot look robust. History persistence is opt-in via --history and is the core's only side effect. Without it the core stays a pure transform. Two committed fixtures (clean and fragile) and new fixture-test sections cover the append-and-trim, both ribbons including the two-arm gap, and the chips.
726 lines
18 KiB
JSON
726 lines
18 KiB
JSON
{
|
|
"skill": "fragile-skill",
|
|
"generatedAt": "2026-07-24T11:00:00Z",
|
|
"temperature": 1.0,
|
|
"trialsPerCase": 5,
|
|
"arms": [
|
|
{
|
|
"id": "new",
|
|
"label": "New skill"
|
|
},
|
|
{
|
|
"id": "previous",
|
|
"label": "Previous version"
|
|
},
|
|
{
|
|
"id": "baseline",
|
|
"label": "No skill"
|
|
}
|
|
],
|
|
"comparisons": [
|
|
{
|
|
"id": "efficacy",
|
|
"label": "Efficacy",
|
|
"new": "new",
|
|
"against": "baseline",
|
|
"rule": "efficacy"
|
|
},
|
|
{
|
|
"id": "regression",
|
|
"label": "Regression",
|
|
"new": "new",
|
|
"against": "previous",
|
|
"rule": "regression"
|
|
}
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "both",
|
|
"description": "both scenario",
|
|
"softCriteria": [
|
|
"a good answer for both"
|
|
],
|
|
"trials": [
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-both-t0",
|
|
"previous": "OUT-previous-both-t0",
|
|
"baseline": "OUT-baseline-both-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-both-t1",
|
|
"previous": "OUT-previous-both-t1",
|
|
"baseline": "OUT-baseline-both-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-both-t2",
|
|
"previous": "OUT-previous-both-t2",
|
|
"baseline": "OUT-baseline-both-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-both-t3",
|
|
"previous": "OUT-previous-both-t3",
|
|
"baseline": "OUT-baseline-both-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "previous",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-both-t4",
|
|
"previous": "OUT-previous-both-t4",
|
|
"baseline": "OUT-baseline-both-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "reg-only",
|
|
"description": "reg-only scenario",
|
|
"softCriteria": [
|
|
"a good answer for reg-only"
|
|
],
|
|
"trials": [
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-reg-only-t0",
|
|
"previous": "OUT-previous-reg-only-t0",
|
|
"baseline": "OUT-baseline-reg-only-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-reg-only-t1",
|
|
"previous": "OUT-previous-reg-only-t1",
|
|
"baseline": "OUT-baseline-reg-only-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-reg-only-t2",
|
|
"previous": "OUT-previous-reg-only-t2",
|
|
"baseline": "OUT-baseline-reg-only-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-reg-only-t3",
|
|
"previous": "OUT-previous-reg-only-t3",
|
|
"baseline": "OUT-baseline-reg-only-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "previous",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-reg-only-t4",
|
|
"previous": "OUT-previous-reg-only-t4",
|
|
"baseline": "OUT-baseline-reg-only-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "none",
|
|
"description": "none scenario",
|
|
"softCriteria": [
|
|
"a good answer for none"
|
|
],
|
|
"trials": [
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-none-t0",
|
|
"previous": "OUT-previous-none-t0",
|
|
"baseline": "OUT-baseline-none-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-none-t1",
|
|
"previous": "OUT-previous-none-t1",
|
|
"baseline": "OUT-baseline-none-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-none-t2",
|
|
"previous": "OUT-previous-none-t2",
|
|
"baseline": "OUT-baseline-none-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-none-t3",
|
|
"previous": "OUT-previous-none-t3",
|
|
"baseline": "OUT-baseline-none-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
},
|
|
{
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
},
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "fixture"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-none-t4",
|
|
"previous": "OUT-previous-none-t4",
|
|
"baseline": "OUT-baseline-none-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 500,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 300,
|
|
"turns": 3
|
|
},
|
|
"previous": {
|
|
"input": 900,
|
|
"output": 450,
|
|
"cacheCreation": 180,
|
|
"cacheRead": 270,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 600,
|
|
"output": 300,
|
|
"cacheCreation": 0,
|
|
"cacheRead": 0,
|
|
"turns": 2
|
|
}
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|