alexion
9b40d9f4e5
Turn a benchmark run into a three-arm experiment: alongside the new-skill and no-skill arms, add a previous-version arm materialized from the default branch's HEAD, drawn automatically whenever the skill's directory differs from HEAD and degrading to the two-arm efficacy-only shape otherwise. Two blind head-to-heads now fall out per trial — efficacy (new-vs-no-skill) and regression (new-vs-old). The deterministic core dispatches a pass rule per comparison (efficacy at wins>=3 and losses<=1, regression at losses<=1 with no wins floor), gates only Efficacy on the hard assertion, and yields a second skill-level Regression verdict, green when no case regressed and not-applicable on a two-arm run. The report gains a second badge, a headline reading both verdicts together, a previous-version cost row with a two-ratio footnote, and per-case side-by-side comparisons that auto-expand on any failure or flagged loss to show the losing-trial output pair. The fixture test covers both shapes in one test: a three-arm run asserting per-arm metrics, both per-case and skill-level verdicts, the net margins, the three-row cost table and losing-trial evidence, and the two-arm run retained as the degenerate no-previous-version case.
955 lines
25 KiB
JSON
955 lines
25 KiB
JSON
{
|
|
"skill": "sample-skill",
|
|
"generatedAt": "2026-07-24T12:00:00Z",
|
|
"temperature": 1.0,
|
|
"trialsPerCase": 5,
|
|
"arms": [
|
|
{
|
|
"id": "new",
|
|
"label": "New skill"
|
|
},
|
|
{
|
|
"id": "previous",
|
|
"label": "Previous version"
|
|
},
|
|
{
|
|
"id": "baseline",
|
|
"label": "No skill"
|
|
}
|
|
],
|
|
"comparisons": [
|
|
{
|
|
"id": "efficacy",
|
|
"label": "Efficacy",
|
|
"new": "new",
|
|
"against": "baseline",
|
|
"rule": "efficacy"
|
|
},
|
|
{
|
|
"id": "regression",
|
|
"label": "Regression",
|
|
"new": "new",
|
|
"against": "previous",
|
|
"rule": "regression"
|
|
}
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "clean-both",
|
|
"description": "New beats baseline and holds steady versus the previous version.",
|
|
"softCriteria": [
|
|
"The answer is grounded in the fixture."
|
|
],
|
|
"trials": [
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-clean-both-t0",
|
|
"previous": "OUT-previous-clean-both-t0",
|
|
"baseline": "OUT-baseline-clean-both-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-clean-both-t1",
|
|
"previous": "OUT-previous-clean-both-t1",
|
|
"baseline": "OUT-baseline-clean-both-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "new",
|
|
"rationale": "judge: regression new"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-clean-both-t2",
|
|
"previous": "OUT-previous-clean-both-t2",
|
|
"baseline": "OUT-baseline-clean-both-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-clean-both-t3",
|
|
"previous": "OUT-previous-clean-both-t3",
|
|
"baseline": "OUT-baseline-clean-both-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "judge: efficacy tie"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-clean-both-t4",
|
|
"previous": "OUT-previous-clean-both-t4",
|
|
"baseline": "OUT-baseline-clean-both-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "regressed-still-valuable",
|
|
"description": "New still beats baseline but has degraded from the previous version.",
|
|
"softCriteria": [
|
|
"The answer resolves the user's request."
|
|
],
|
|
"trials": [
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "previous",
|
|
"rationale": "judge: regression previous"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-regressed-still-valuable-t0",
|
|
"previous": "OUT-previous-regressed-still-valuable-t0",
|
|
"baseline": "OUT-baseline-regressed-still-valuable-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "previous",
|
|
"rationale": "judge: regression previous"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-regressed-still-valuable-t1",
|
|
"previous": "OUT-previous-regressed-still-valuable-t1",
|
|
"baseline": "OUT-baseline-regressed-still-valuable-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-regressed-still-valuable-t2",
|
|
"previous": "OUT-previous-regressed-still-valuable-t2",
|
|
"baseline": "OUT-baseline-regressed-still-valuable-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "judge: efficacy tie"
|
|
},
|
|
"regression": {
|
|
"winner": "new",
|
|
"rationale": "judge: regression new"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-regressed-still-valuable-t3",
|
|
"previous": "OUT-previous-regressed-still-valuable-t3",
|
|
"baseline": "OUT-baseline-regressed-still-valuable-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-regressed-still-valuable-t4",
|
|
"previous": "OUT-previous-regressed-still-valuable-t4",
|
|
"baseline": "OUT-baseline-regressed-still-valuable-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "dead-weight",
|
|
"description": "New does not beat baseline, but the edit did not make it worse.",
|
|
"softCriteria": [
|
|
"The answer is correct."
|
|
],
|
|
"trials": [
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "judge: efficacy tie"
|
|
},
|
|
"regression": {
|
|
"winner": "new",
|
|
"rationale": "judge: regression new"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-dead-weight-t0",
|
|
"previous": "OUT-previous-dead-weight-t0",
|
|
"baseline": "OUT-baseline-dead-weight-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "judge: efficacy tie"
|
|
},
|
|
"regression": {
|
|
"winner": "new",
|
|
"rationale": "judge: regression new"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-dead-weight-t1",
|
|
"previous": "OUT-previous-dead-weight-t1",
|
|
"baseline": "OUT-baseline-dead-weight-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "baseline",
|
|
"rationale": "judge: efficacy baseline"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-dead-weight-t2",
|
|
"previous": "OUT-previous-dead-weight-t2",
|
|
"baseline": "OUT-baseline-dead-weight-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "judge: efficacy tie"
|
|
},
|
|
"regression": {
|
|
"winner": "new",
|
|
"rationale": "judge: regression new"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-dead-weight-t3",
|
|
"previous": "OUT-previous-dead-weight-t3",
|
|
"baseline": "OUT-baseline-dead-weight-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "tie",
|
|
"rationale": "judge: efficacy tie"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-dead-weight-t4",
|
|
"previous": "OUT-previous-dead-weight-t4",
|
|
"baseline": "OUT-baseline-dead-weight-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "hard-gate-fail",
|
|
"description": "New sweeps both head-to-heads but violates a hard assertion.",
|
|
"softCriteria": [
|
|
"The output is well-formed."
|
|
],
|
|
"trials": [
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-hard-gate-fail-t0",
|
|
"previous": "OUT-previous-hard-gate-fail-t0",
|
|
"baseline": "OUT-baseline-hard-gate-fail-t0"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-hard-gate-fail-t1",
|
|
"previous": "OUT-previous-hard-gate-fail-t1",
|
|
"baseline": "OUT-baseline-hard-gate-fail-t1"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-hard-gate-fail-t2",
|
|
"previous": "OUT-previous-hard-gate-fail-t2",
|
|
"baseline": "OUT-baseline-hard-gate-fail-t2"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": false
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-hard-gate-fail-t3",
|
|
"previous": "OUT-previous-hard-gate-fail-t3",
|
|
"baseline": "OUT-baseline-hard-gate-fail-t3"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
},
|
|
{
|
|
"comparisons": {
|
|
"efficacy": {
|
|
"winner": "new",
|
|
"rationale": "judge: efficacy new"
|
|
},
|
|
"regression": {
|
|
"winner": "tie",
|
|
"rationale": "judge: regression tie"
|
|
}
|
|
},
|
|
"outputs": {
|
|
"new": "OUT-new-hard-gate-fail-t4",
|
|
"previous": "OUT-previous-hard-gate-fail-t4",
|
|
"baseline": "OUT-baseline-hard-gate-fail-t4"
|
|
},
|
|
"usage": {
|
|
"new": {
|
|
"input": 1000,
|
|
"output": 2000,
|
|
"cacheCreation": 400,
|
|
"cacheRead": 8000,
|
|
"turns": 3
|
|
},
|
|
"baseline": {
|
|
"input": 500,
|
|
"output": 1000,
|
|
"cacheCreation": 200,
|
|
"cacheRead": 4000,
|
|
"turns": 2
|
|
},
|
|
"previous": {
|
|
"input": 800,
|
|
"output": 1600,
|
|
"cacheCreation": 300,
|
|
"cacheRead": 6000,
|
|
"turns": 3
|
|
}
|
|
},
|
|
"hard": {
|
|
"ran": true,
|
|
"pass": true
|
|
}
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|