alexion/skills
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add trend history and ribbons (task 0006)

Browse Source 
Add the longitudinal layer to the benchmark core: a per-skill JSON-lines
history that appends one summary line per run and trims oldest-first at
fifty, two stacked net-margin trend ribbons (Efficacy and Regression, the
latter leaving a gap for two-arm runs) with the current run ringed and a
net/delta/green-count readout, and per-badge fragility chips that flag the
narrowest passing case on each green axis so a barely-green skill cannot
look robust.

History persistence is opt-in via --history and is the core's only side
effect. Without it the core stays a pure transform. Two committed fixtures
(clean and fragile) and new fixture-test sections cover the append-and-trim,
both ribbons including the two-arm gap, and the chips.
This commit was merged in pull request #6.
This commit is contained in:
alexion
2026-07-24 16:37:11 -04:00
parent 9b40d9f4e5
commit f4df5d31c9
6 changed files with 1645 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
checks/benchmark-core.nix
View File

@@ -3,6 +3,9 @@
# skill-level Efficacy and Regression verdicts, and the rendered report — for
# both the three-arm shape and the degenerate two-arm "no previous version"
# shape.
# It also drives the longitudinal layer: the per-skill history append-and-trim,
# the two trend ribbons including the two-arm gap in the regression series, and
# the per-badge fragility chips on a clean run and a green-but-fragile one.
# No LLM runs.
{
pkgs,
@@ -11,11 +14,13 @@ let
core = ../skills/benchmark-skill/core/benchmark_core.py;
threeArm = ./fixtures/benchmark/three-arm-bundle.json;
twoArm = ./fixtures/benchmark/two-arm-bundle.json;
clean = ./fixtures/benchmark/clean-bundle.json;
fragile = ./fixtures/benchmark/fragile-bundle.json;
in
pkgs.runCommandLocal "benchmark-core-check"
{
nativeBuildInputs = [ pkgs.python3 ];
inherit core threeArm twoArm;
inherit core threeArm twoArm clean fragile;
}
''
fail() { echo "FAIL: $1" >&2; exit 1; }
@@ -180,5 +185,97 @@ pkgs.runCommandLocal "benchmark-core-check"
grep -q 'OUT-new-regresses-baseline-t3' two.html || fail "missing new evidence for the two-arm efficacy loss"
grep -q 'OUT-baseline-regresses-baseline-t3' two.html || fail "missing baseline evidence for the two-arm efficacy loss"
# --- clean run: fully collapsed, one narrowest-margin chip, ribbons ---------
echo "the core rests a clean run fully collapsed with a lone narrowest-margin chip"
python3 "$core" "$clean" --json clean.json --html clean.html --history clean.history.jsonl \
|| fail "the core exited non-zero on the clean bundle"
python3 - clean.json <<'PY' || fail "a clean-run assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["efficacyVerdict"] == "green", r["efficacyVerdict"]
assert r["regressionVerdict"] == "green", r["regressionVerdict"]
cases = {c["name"]: c for c in r["cases"]}
# The narrowest efficacy case (fewest wins) carries the chip, the roomy one does not.
assert [ch["axis"] for ch in cases["narrowest"]["chips"]] == ["efficacy"], cases["narrowest"]["chips"]
assert cases["roomy"]["chips"] == [], cases["roomy"]["chips"]
print("clean-run assertions passed")
PY
if grep -q ' open>' clean.html; then fail "a clean run did not rest fully collapsed"; fi
grep -q 'class="chip efficacy"' clean.html || fail "clean run omits the narrowest-margin efficacy chip"
if grep -q 'class="chip regression"' clean.html; then fail "clean run shows a spurious regression chip"; fi
grep -q 'class="ribbons"' clean.html || fail "clean run omits the trend ribbons"
grep -q 'class="spark"' clean.html || fail "clean run omits a sparkline"
grep -q 'class="ring"' clean.html || fail "clean run does not ring the current run"
# --- history append-and-trim and the two-ribbon window with a two-arm gap ---
echo "the core appends to and trims the per-skill history and plots the ribbon window"
python3 - seed.history.jsonl <<'PY' || fail "seeding the history fixture failed"
import json, sys
# 55 prior runs, all three-arm except the most recent, which is two-arm and so
# must plot a gap on the regression axis once it lands inside the 7-run window.
lines = []
for i in range(55):
two = i == 54
lines.append({
"generatedAt": f"seed-{i}",
"armShape": "two-arm" if two else "three-arm",
"efficacyNet": i % 5,
"efficacyPass": True,
"regressionNet": None if two else i % 3,
"regressionPass": None if two else True,
})
open(sys.argv[1], "w").write("".join(json.dumps(l) + "\n" for l in lines))
PY
python3 "$core" "$clean" --json trend.json --history seed.history.jsonl \
|| fail "the core exited non-zero on the seeded-history run"
python3 - seed.history.jsonl trend.json <<'PY' || fail "a history/trend assertion failed"
import json, sys
lines = [json.loads(l) for l in open(sys.argv[1]) if l.strip()]
# 55 prior + this run = 56, trimmed oldest-first back to the cap of 50.
assert len(lines) == 50, len(lines)
gens = [l["generatedAt"] for l in lines]
assert "seed-0" not in gens and "seed-5" not in gens, "oldest runs were not trimmed"
assert "seed-6" in gens and "seed-54" in gens, "recent runs were wrongly trimmed"
assert lines[-1]["armShape"] == "three-arm", lines[-1]
assert lines[-1]["efficacyNet"] == 8 and lines[-1]["regressionNet"] == 1, lines[-1]
t = json.load(open(sys.argv[2]))["trend"]
eff, reg = t["efficacy"]["points"], t["regression"]["points"]
assert len(eff) == 7 and len(reg) == 7, (len(eff), len(reg))
# Efficacy has a value every run, while regression breaks at the two-arm run.
assert all(p["net"] is not None for p in eff), "efficacy series should have no gaps"
assert eff[-1]["current"] and reg[-1]["current"], "the current run is ringed on both axes"
assert reg[5]["net"] is None, [p["net"] for p in reg]
assert reg[-1]["net"] is not None, "the current three-arm run has a regression dot"
assert t["regression"]["applicable"] == 6, t["regression"]["applicable"]
print("history/trend assertions passed")
PY
# --- green-but-fragile run: independent per-badge chips, both on one case ----
echo "the core chips each fragility axis independently, and both on a doubly-fragile case"
python3 "$core" "$fragile" --json fragile.json --html fragile.html \
|| fail "the core exited non-zero on the fragile bundle"
python3 - fragile.json <<'PY' || fail "a fragility-chip assertion failed"
import json, sys
r = json.load(open(sys.argv[1]))
assert r["efficacyVerdict"] == "green" and r["regressionVerdict"] == "green"
cases = {c["name"]: c for c in r["cases"]}
# One case is both the narrowest efficacy margin and at exactly one regression
# loss, so it carries both chips.
# The others carry at most their own axis.
assert {ch["axis"] for ch in cases["both"]["chips"]} == {"efficacy", "regression"}, cases["both"]["chips"]
assert [ch["axis"] for ch in cases["reg-only"]["chips"]] == ["regression"], cases["reg-only"]["chips"]
assert cases["none"]["chips"] == [], cases["none"]["chips"]
print("fragility-chip assertions passed")
PY
grep -q 'class="chip efficacy"' fragile.html || fail "fragile run omits the efficacy chip"
grep -q 'class="chip regression"' fragile.html || fail "fragile run omits the regression chip"
touch "$out"
''

496
checks/fixtures/benchmark/clean-bundle.json Normal file
View File

@@ -0,0 +1,496 @@
{
"skill": "clean-skill",
"generatedAt": "2026-07-24T10:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "narrowest",
"description": "narrowest scenario",
"softCriteria": [
"a good answer for narrowest"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t0",
"previous": "OUT-previous-narrowest-t0",
"baseline": "OUT-baseline-narrowest-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t1",
"previous": "OUT-previous-narrowest-t1",
"baseline": "OUT-baseline-narrowest-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t2",
"previous": "OUT-previous-narrowest-t2",
"baseline": "OUT-baseline-narrowest-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t3",
"previous": "OUT-previous-narrowest-t3",
"baseline": "OUT-baseline-narrowest-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-narrowest-t4",
"previous": "OUT-previous-narrowest-t4",
"baseline": "OUT-baseline-narrowest-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "roomy",
"description": "roomy scenario",
"softCriteria": [
"a good answer for roomy"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "new",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t0",
"previous": "OUT-previous-roomy-t0",
"baseline": "OUT-baseline-roomy-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t1",
"previous": "OUT-previous-roomy-t1",
"baseline": "OUT-baseline-roomy-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t2",
"previous": "OUT-previous-roomy-t2",
"baseline": "OUT-baseline-roomy-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t3",
"previous": "OUT-previous-roomy-t3",
"baseline": "OUT-baseline-roomy-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-roomy-t4",
"previous": "OUT-previous-roomy-t4",
"baseline": "OUT-baseline-roomy-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
}
]
}

725
checks/fixtures/benchmark/fragile-bundle.json Normal file
View File

@@ -0,0 +1,725 @@
{
"skill": "fragile-skill",
"generatedAt": "2026-07-24T11:00:00Z",
"temperature": 1.0,
"trialsPerCase": 5,
"arms": [
{
"id": "new",
"label": "New skill"
},
{
"id": "previous",
"label": "Previous version"
},
{
"id": "baseline",
"label": "No skill"
}
],
"comparisons": [
{
"id": "efficacy",
"label": "Efficacy",
"new": "new",
"against": "baseline",
"rule": "efficacy"
},
{
"id": "regression",
"label": "Regression",
"new": "new",
"against": "previous",
"rule": "regression"
}
],
"cases": [
{
"name": "both",
"description": "both scenario",
"softCriteria": [
"a good answer for both"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "new",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t0",
"previous": "OUT-previous-both-t0",
"baseline": "OUT-baseline-both-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t1",
"previous": "OUT-previous-both-t1",
"baseline": "OUT-baseline-both-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t2",
"previous": "OUT-previous-both-t2",
"baseline": "OUT-baseline-both-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t3",
"previous": "OUT-previous-both-t3",
"baseline": "OUT-baseline-both-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "previous",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-both-t4",
"previous": "OUT-previous-both-t4",
"baseline": "OUT-baseline-both-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "reg-only",
"description": "reg-only scenario",
"softCriteria": [
"a good answer for reg-only"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t0",
"previous": "OUT-previous-reg-only-t0",
"baseline": "OUT-baseline-reg-only-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t1",
"previous": "OUT-previous-reg-only-t1",
"baseline": "OUT-baseline-reg-only-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t2",
"previous": "OUT-previous-reg-only-t2",
"baseline": "OUT-baseline-reg-only-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t3",
"previous": "OUT-previous-reg-only-t3",
"baseline": "OUT-baseline-reg-only-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "previous",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-reg-only-t4",
"previous": "OUT-previous-reg-only-t4",
"baseline": "OUT-baseline-reg-only-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
},
{
"name": "none",
"description": "none scenario",
"softCriteria": [
"a good answer for none"
],
"trials": [
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t0",
"previous": "OUT-previous-none-t0",
"baseline": "OUT-baseline-none-t0"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t1",
"previous": "OUT-previous-none-t1",
"baseline": "OUT-baseline-none-t1"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t2",
"previous": "OUT-previous-none-t2",
"baseline": "OUT-baseline-none-t2"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "new",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t3",
"previous": "OUT-previous-none-t3",
"baseline": "OUT-baseline-none-t3"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
},
{
"hard": {
"ran": true,
"pass": true
},
"comparisons": {
"efficacy": {
"winner": "tie",
"rationale": "fixture"
},
"regression": {
"winner": "tie",
"rationale": "fixture"
}
},
"outputs": {
"new": "OUT-new-none-t4",
"previous": "OUT-previous-none-t4",
"baseline": "OUT-baseline-none-t4"
},
"usage": {
"new": {
"input": 1000,
"output": 500,
"cacheCreation": 200,
"cacheRead": 300,
"turns": 3
},
"previous": {
"input": 900,
"output": 450,
"cacheCreation": 180,
"cacheRead": 270,
"turns": 3
},
"baseline": {
"input": 600,
"output": 300,
"cacheCreation": 0,
"cacheRead": 0,
"turns": 2
}
}
}
]
}
]
}