feat: add regression arm and two verdicts (task 0005)
Turn a benchmark run into a three-arm experiment: alongside the new-skill and no-skill arms, add a previous-version arm materialized from the default branch's HEAD, drawn automatically whenever the skill's directory differs from HEAD and degrading to the two-arm efficacy-only shape otherwise. Two blind head-to-heads now fall out per trial — efficacy (new-vs-no-skill) and regression (new-vs-old). The deterministic core dispatches a pass rule per comparison (efficacy at wins>=3 and losses<=1, regression at losses<=1 with no wins floor), gates only Efficacy on the hard assertion, and yields a second skill-level Regression verdict, green when no case regressed and not-applicable on a two-arm run. The report gains a second badge, a headline reading both verdicts together, a previous-version cost row with a two-ratio footnote, and per-case side-by-side comparisons that auto-expand on any failure or flagged loss to show the losing-trial output pair. The fixture test covers both shapes in one test: a three-arm run asserting per-arm metrics, both per-case and skill-level verdicts, the net margins, the three-row cost table and losing-trial evidence, and the two-arm run retained as the degenerate no-previous-version case.
This commit was merged in pull request #5.
This commit is contained in:
@@ -1,34 +1,146 @@
|
||||
# Feeds the committed run-bundle fixture through the benchmark skill's
|
||||
# Feeds the committed run-bundle fixtures through the benchmark skill's
|
||||
# deterministic core and asserts the per-arm metrics, the per-case and
|
||||
# skill-level Efficacy verdict, and the rendered report.
|
||||
# skill-level Efficacy and Regression verdicts, and the rendered report — for
|
||||
# both the three-arm shape and the degenerate two-arm "no previous version"
|
||||
# shape.
|
||||
# No LLM runs.
|
||||
{
|
||||
pkgs,
|
||||
}:
|
||||
let
|
||||
core = ../skills/benchmark-skill/core/benchmark_core.py;
|
||||
fixture = ./fixtures/benchmark/run-bundle.json;
|
||||
threeArm = ./fixtures/benchmark/three-arm-bundle.json;
|
||||
twoArm = ./fixtures/benchmark/two-arm-bundle.json;
|
||||
in
|
||||
pkgs.runCommandLocal "benchmark-core-check"
|
||||
{
|
||||
nativeBuildInputs = [ pkgs.python3 ];
|
||||
inherit core fixture;
|
||||
inherit core threeArm twoArm;
|
||||
}
|
||||
''
|
||||
fail() { echo "FAIL: $1" >&2; exit 1; }
|
||||
|
||||
echo "the core turns a run bundle into a results model and an HTML report"
|
||||
python3 "$core" "$fixture" --json results.json --html report.html \
|
||||
|| fail "the core exited non-zero"
|
||||
# --- three-arm run --------------------------------------------------------
|
||||
echo "the core scores a three-arm run and renders its report"
|
||||
python3 "$core" "$threeArm" --json three.json --html three.html \
|
||||
|| fail "the core exited non-zero on the three-arm bundle"
|
||||
|
||||
echo "the results model carries the expected metrics and verdicts"
|
||||
python3 - results.json <<'PY' || fail "a results-model assertion failed"
|
||||
echo "the three-arm results model carries both verdicts, net margins, and metrics"
|
||||
python3 - three.json <<'PY' || fail "a three-arm results-model assertion failed"
|
||||
import json, sys
|
||||
r = json.load(open(sys.argv[1]))
|
||||
|
||||
assert r["armShape"] == "three-arm", r["armShape"]
|
||||
# Efficacy is red: dead-weight fails the wins floor and hard-gate-fail trips
|
||||
# the gate. Regression is red: regressed-still-valuable loses twice to the
|
||||
# previous version.
|
||||
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
|
||||
assert r["regressionVerdict"] == "red", r["regressionVerdict"]
|
||||
assert r["efficacyNetMargin"] == 12, r["efficacyNetMargin"]
|
||||
assert r["regressionNetMargin"] == 3, r["regressionNetMargin"]
|
||||
|
||||
new = r["armMetrics"]["new"]
|
||||
assert new["turns"] == 60, new["turns"]
|
||||
assert new["rawTokens"] == 228000, new["rawTokens"]
|
||||
assert new["costEquivalentTokens"] == 246000, new["costEquivalentTokens"]
|
||||
assert abs(new["imputedCost"] - 1.23) < 1e-9, new["imputedCost"]
|
||||
|
||||
base = r["armMetrics"]["baseline"]
|
||||
assert base["turns"] == 40, base["turns"]
|
||||
assert base["rawTokens"] == 114000, base["rawTokens"]
|
||||
assert base["costEquivalentTokens"] == 123000, base["costEquivalentTokens"]
|
||||
|
||||
prev = r["armMetrics"]["previous"]
|
||||
assert prev["turns"] == 60, prev["turns"]
|
||||
assert prev["rawTokens"] == 174000, prev["rawTokens"]
|
||||
assert prev["costEquivalentTokens"] == 195500, prev["costEquivalentTokens"]
|
||||
assert abs(prev["imputedCost"] - 0.9775) < 1e-9, prev["imputedCost"]
|
||||
|
||||
# Cost-table rows render in arm order: new-skill, previous-version, no-skill.
|
||||
assert [a["id"] for a in r["arms"]] == ["new", "previous", "baseline"], \
|
||||
[a["id"] for a in r["arms"]]
|
||||
|
||||
cases = {c["name"]: c for c in r["cases"]}
|
||||
# Authored order is preserved, never reshuffled by verdict.
|
||||
assert [c["name"] for c in r["cases"]] == [
|
||||
"clean-both", "regressed-still-valuable", "dead-weight", "hard-gate-fail"
|
||||
]
|
||||
|
||||
a = cases["clean-both"]
|
||||
assert a["efficacyPassed"] is True
|
||||
assert a["regressionPassed"] is True
|
||||
assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"]
|
||||
assert a["comparisons"]["regression"]["trials"] == ["TIE", "TIE", "WIN", "TIE", "TIE"]
|
||||
|
||||
# Efficacy green but regression red — the exact crossing this feature exists
|
||||
# to catch: still beats no-skill, yet degraded from the released version.
|
||||
b = cases["regressed-still-valuable"]
|
||||
assert b["efficacyPassed"] is True
|
||||
assert b["regressionPassed"] is False
|
||||
assert b["comparisons"]["regression"]["losses"] == 2
|
||||
assert b["comparisons"]["regression"]["flaggedLosses"] == [0, 1]
|
||||
|
||||
# Efficacy red but regression green — already dead weight, but the edit did
|
||||
# not make it worse.
|
||||
c = cases["dead-weight"]
|
||||
assert c["efficacyPassed"] is False
|
||||
assert c["regressionPassed"] is True
|
||||
assert c["comparisons"]["efficacy"]["wins"] == 0
|
||||
assert c["comparisons"]["efficacy"]["losses"] == 1
|
||||
assert c["comparisons"]["efficacy"]["flaggedLosses"] == [2]
|
||||
assert c["comparisons"]["regression"]["wins"] == 3
|
||||
|
||||
# The hard gate fails efficacy outright but does not gate regression, which
|
||||
# is judged purely on losses.
|
||||
d = cases["hard-gate-fail"]
|
||||
assert d["hardFailed"] is True
|
||||
assert d["efficacyPassed"] is False
|
||||
assert d["regressionPassed"] is True
|
||||
assert d["comparisons"]["efficacy"]["wins"] == 5
|
||||
print("three-arm results-model assertions passed")
|
||||
PY
|
||||
|
||||
echo "the three-arm report shows both badges, the three-row table, and evidence"
|
||||
grep -q '<!doctype html>' three.html || fail "three-arm report is not self-contained"
|
||||
grep -q 'Efficacy: RED' three.html || fail "three-arm report is missing the red Efficacy badge"
|
||||
grep -q 'Regression: RED' three.html || fail "three-arm report is missing the red Regression badge"
|
||||
grep -qi 'regress' three.html || fail "three-arm report is missing the verdict headline"
|
||||
grep -q 'Cost-equiv tokens' three.html || fail "three-arm report is missing the cost table"
|
||||
grep -q 'Previous version' three.html || fail "three-arm report is missing the previous-version row"
|
||||
grep -q 'shared-context cache' three.html || fail "three-arm report is missing the cache footnote"
|
||||
grep -q 'new-vs-no-skill' three.html || fail "three-arm footnote omits the new-vs-no-skill ratio"
|
||||
grep -q 'new-vs-old' three.html || fail "three-arm footnote omits the new-vs-old ratio"
|
||||
for name in clean-both regressed-still-valuable dead-weight hard-gate-fail; do
|
||||
grep -q "$name" three.html || fail "three-arm report omits case $name"
|
||||
done
|
||||
grep -q 'class="cell LOSS"' three.html || fail "three-arm report is missing a LOSS cell"
|
||||
grep -q 'Hard assertion failed' three.html || fail "three-arm report does not flag the hard failure"
|
||||
grep -q ' open>' three.html || fail "three-arm report does not auto-expand a failing case"
|
||||
grep -q 'class="cmps"' three.html || fail "three-arm report does not lay comparisons side by side"
|
||||
# Evidence for the failed regression comparison: new vs previous on trial 1.
|
||||
grep -q 'OUT-new-regressed-still-valuable-t0' three.html || fail "missing new evidence for the regression loss"
|
||||
grep -q 'OUT-previous-regressed-still-valuable-t0' three.html || fail "missing previous evidence for the regression loss"
|
||||
# Evidence for the failed efficacy comparison: new vs baseline on trial 3.
|
||||
grep -q 'OUT-new-dead-weight-t2' three.html || fail "missing new evidence for the efficacy loss"
|
||||
grep -q 'OUT-baseline-dead-weight-t2' three.html || fail "missing baseline evidence for the efficacy loss"
|
||||
# A clean, collapsed case emits no losing-trial evidence.
|
||||
if grep -q 'OUT-new-clean-both' three.html; then fail "a clean case leaked losing-trial evidence"; fi
|
||||
|
||||
# --- two-arm run ----------------------------------------------------------
|
||||
echo "the core scores the degenerate two-arm run"
|
||||
python3 "$core" "$twoArm" --json two.json --html two.html \
|
||||
|| fail "the core exited non-zero on the two-arm bundle"
|
||||
|
||||
echo "the two-arm results model reads regression as not-applicable"
|
||||
python3 - two.json <<'PY' || fail "a two-arm results-model assertion failed"
|
||||
import json, sys
|
||||
r = json.load(open(sys.argv[1]))
|
||||
|
||||
assert r["armShape"] == "two-arm", r["armShape"]
|
||||
# The skill verdict is red because two of the three cases fail efficacy.
|
||||
assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
|
||||
assert r["regressionVerdict"] == "not-applicable", r["regressionVerdict"]
|
||||
assert r["regressionNetMargin"] is None, r["regressionNetMargin"]
|
||||
assert r["efficacyNetMargin"] == 9, r["efficacyNetMargin"]
|
||||
|
||||
new = r["armMetrics"]["new"]
|
||||
assert new["turns"] == 45, new["turns"]
|
||||
@@ -41,41 +153,32 @@ pkgs.runCommandLocal "benchmark-core-check"
|
||||
assert base["rawTokens"] == 85500, base["rawTokens"]
|
||||
assert base["costEquivalentTokens"] == 92250, base["costEquivalentTokens"]
|
||||
|
||||
cases = {c["name"]: c for c in r["cases"]}
|
||||
# Authored order is preserved, never reshuffled by verdict.
|
||||
assert [c["name"] for c in r["cases"]] == ["clean-pass", "regresses-baseline", "hard-gate-fail"]
|
||||
assert "previous" not in r["armMetrics"], "two-arm run has no previous arm"
|
||||
|
||||
a = cases["clean-pass"]
|
||||
assert a["efficacyPassed"] is True
|
||||
assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"]
|
||||
assert a["comparisons"]["efficacy"]["wins"] == 4
|
||||
assert a["comparisons"]["efficacy"]["losses"] == 0
|
||||
assert [c["name"] for c in r["cases"]] == [
|
||||
"clean-pass", "regresses-baseline", "hard-gate-fail"
|
||||
]
|
||||
for c in r["cases"]:
|
||||
assert c["regressionPassed"] is None, (c["name"], c["regressionPassed"])
|
||||
|
||||
# A head-to-head fail: two losses drop it under the wins>=3, losses<=1 rule,
|
||||
# and both losses are flagged for human review.
|
||||
b = cases["regresses-baseline"]
|
||||
b = {c["name"]: c for c in r["cases"]}["regresses-baseline"]
|
||||
assert b["efficacyPassed"] is False
|
||||
assert b["comparisons"]["efficacy"]["losses"] == 2
|
||||
assert b["comparisons"]["efficacy"]["flaggedLosses"] == [3, 4]
|
||||
|
||||
# A hard-assertion failure fails the case outright despite a clean sweep.
|
||||
c = cases["hard-gate-fail"]
|
||||
assert c["hardFailed"] is True
|
||||
assert c["efficacyPassed"] is False
|
||||
assert c["comparisons"]["efficacy"]["wins"] == 5
|
||||
print("results-model assertions passed")
|
||||
h = {c["name"]: c for c in r["cases"]}["hard-gate-fail"]
|
||||
assert h["hardFailed"] is True
|
||||
print("two-arm results-model assertions passed")
|
||||
PY
|
||||
|
||||
echo "the report is self-contained and shows the badge, cost table, and cases"
|
||||
grep -q '<!doctype html>' report.html || fail "report is not a self-contained document"
|
||||
grep -q 'Efficacy: RED' report.html || fail "report is missing the red Efficacy badge"
|
||||
grep -q 'Cost-equiv tokens' report.html || fail "report is missing the per-arm cost table"
|
||||
grep -q 'shared-context cache' report.html || fail "report is missing the cache-overhead footnote"
|
||||
for name in clean-pass regresses-baseline hard-gate-fail; do
|
||||
grep -q "$name" report.html || fail "report omits case $name"
|
||||
done
|
||||
grep -q 'class="cell LOSS"' report.html || fail "report is missing a per-trial LOSS cell"
|
||||
grep -q 'Hard assertion failed' report.html || fail "report does not flag the hard-assertion failure"
|
||||
echo "the two-arm report reads Regression as not-applicable and drops the previous row"
|
||||
grep -q 'Efficacy: RED' two.html || fail "two-arm report is missing the Efficacy badge"
|
||||
grep -q 'Regression: N/A' two.html || fail "two-arm report does not read Regression as not-applicable"
|
||||
grep -q 'new-vs-no-skill' two.html || fail "two-arm footnote omits the new-vs-no-skill ratio"
|
||||
if grep -q 'new-vs-old' two.html; then fail "two-arm footnote names a new-vs-old ratio that does not apply"; fi
|
||||
if grep -q 'Previous version' two.html; then fail "two-arm report shows a previous-version row"; fi
|
||||
grep -q 'OUT-new-regresses-baseline-t3' two.html || fail "missing new evidence for the two-arm efficacy loss"
|
||||
grep -q 'OUT-baseline-regresses-baseline-t3' two.html || fail "missing baseline evidence for the two-arm efficacy loss"
|
||||
|
||||
touch "$out"
|
||||
''
|
||||
|
||||
954
checks/fixtures/benchmark/three-arm-bundle.json
Normal file
954
checks/fixtures/benchmark/three-arm-bundle.json
Normal file
@@ -0,0 +1,954 @@
|
||||
{
|
||||
"skill": "sample-skill",
|
||||
"generatedAt": "2026-07-24T12:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
"arms": [
|
||||
{
|
||||
"id": "new",
|
||||
"label": "New skill"
|
||||
},
|
||||
{
|
||||
"id": "previous",
|
||||
"label": "Previous version"
|
||||
},
|
||||
{
|
||||
"id": "baseline",
|
||||
"label": "No skill"
|
||||
}
|
||||
],
|
||||
"comparisons": [
|
||||
{
|
||||
"id": "efficacy",
|
||||
"label": "Efficacy",
|
||||
"new": "new",
|
||||
"against": "baseline",
|
||||
"rule": "efficacy"
|
||||
},
|
||||
{
|
||||
"id": "regression",
|
||||
"label": "Regression",
|
||||
"new": "new",
|
||||
"against": "previous",
|
||||
"rule": "regression"
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "clean-both",
|
||||
"description": "New beats baseline and holds steady versus the previous version.",
|
||||
"softCriteria": [
|
||||
"The answer is grounded in the fixture."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t0",
|
||||
"previous": "OUT-previous-clean-both-t0",
|
||||
"baseline": "OUT-baseline-clean-both-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t1",
|
||||
"previous": "OUT-previous-clean-both-t1",
|
||||
"baseline": "OUT-baseline-clean-both-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t2",
|
||||
"previous": "OUT-previous-clean-both-t2",
|
||||
"baseline": "OUT-baseline-clean-both-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t3",
|
||||
"previous": "OUT-previous-clean-both-t3",
|
||||
"baseline": "OUT-baseline-clean-both-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t4",
|
||||
"previous": "OUT-previous-clean-both-t4",
|
||||
"baseline": "OUT-baseline-clean-both-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "regressed-still-valuable",
|
||||
"description": "New still beats baseline but has degraded from the previous version.",
|
||||
"softCriteria": [
|
||||
"The answer resolves the user's request."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "previous",
|
||||
"rationale": "judge: regression previous"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t0",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t0",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "previous",
|
||||
"rationale": "judge: regression previous"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t1",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t1",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t2",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t2",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t3",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t3",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t4",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t4",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "dead-weight",
|
||||
"description": "New does not beat baseline, but the edit did not make it worse.",
|
||||
"softCriteria": [
|
||||
"The answer is correct."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t0",
|
||||
"previous": "OUT-previous-dead-weight-t0",
|
||||
"baseline": "OUT-baseline-dead-weight-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t1",
|
||||
"previous": "OUT-previous-dead-weight-t1",
|
||||
"baseline": "OUT-baseline-dead-weight-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge: efficacy baseline"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t2",
|
||||
"previous": "OUT-previous-dead-weight-t2",
|
||||
"baseline": "OUT-baseline-dead-weight-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t3",
|
||||
"previous": "OUT-previous-dead-weight-t3",
|
||||
"baseline": "OUT-baseline-dead-weight-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t4",
|
||||
"previous": "OUT-previous-dead-weight-t4",
|
||||
"baseline": "OUT-baseline-dead-weight-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hard-gate-fail",
|
||||
"description": "New sweeps both head-to-heads but violates a hard assertion.",
|
||||
"softCriteria": [
|
||||
"The output is well-formed."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t0",
|
||||
"previous": "OUT-previous-hard-gate-fail-t0",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t1",
|
||||
"previous": "OUT-previous-hard-gate-fail-t1",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t2",
|
||||
"previous": "OUT-previous-hard-gate-fail-t2",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t3",
|
||||
"previous": "OUT-previous-hard-gate-fail-t3",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t4",
|
||||
"previous": "OUT-previous-hard-gate-fail-t4",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"skill": "sample-skill",
|
||||
"skill": "sample-skill-2arm",
|
||||
"generatedAt": "2026-07-24T12:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
@@ -35,9 +35,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t0",
|
||||
"baseline": "OUT-baseline-clean-pass-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -63,9 +67,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t1",
|
||||
"baseline": "OUT-baseline-clean-pass-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -91,9 +99,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t2",
|
||||
"baseline": "OUT-baseline-clean-pass-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -119,9 +131,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t3",
|
||||
"baseline": "OUT-baseline-clean-pass-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -147,9 +163,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge preferred tie"
|
||||
"rationale": "judge: efficacy tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t4",
|
||||
"baseline": "OUT-baseline-clean-pass-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -184,9 +204,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t0",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -208,9 +232,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t1",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -232,9 +260,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge preferred tie"
|
||||
"rationale": "judge: efficacy tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t2",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -256,9 +288,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge preferred baseline"
|
||||
"rationale": "judge: efficacy baseline"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t3",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -280,9 +316,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge preferred baseline"
|
||||
"rationale": "judge: efficacy baseline"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t4",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -313,9 +353,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t0",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -341,9 +385,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t1",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -369,9 +417,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t2",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -397,9 +449,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t3",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
@@ -425,9 +481,13 @@
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge preferred new"
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t4",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
Reference in New Issue
Block a user