Remove benchmark skill
This commit is contained in:
@@ -1,496 +0,0 @@
|
||||
{
|
||||
"skill": "clean-skill",
|
||||
"generatedAt": "2026-07-24T10:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
"arms": [
|
||||
{
|
||||
"id": "new",
|
||||
"label": "New skill"
|
||||
},
|
||||
{
|
||||
"id": "previous",
|
||||
"label": "Previous version"
|
||||
},
|
||||
{
|
||||
"id": "baseline",
|
||||
"label": "No skill"
|
||||
}
|
||||
],
|
||||
"comparisons": [
|
||||
{
|
||||
"id": "efficacy",
|
||||
"label": "Efficacy",
|
||||
"new": "new",
|
||||
"against": "baseline",
|
||||
"rule": "efficacy"
|
||||
},
|
||||
{
|
||||
"id": "regression",
|
||||
"label": "Regression",
|
||||
"new": "new",
|
||||
"against": "previous",
|
||||
"rule": "regression"
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "narrowest",
|
||||
"description": "narrowest scenario",
|
||||
"softCriteria": [
|
||||
"a good answer for narrowest"
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-narrowest-t0",
|
||||
"previous": "OUT-previous-narrowest-t0",
|
||||
"baseline": "OUT-baseline-narrowest-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-narrowest-t1",
|
||||
"previous": "OUT-previous-narrowest-t1",
|
||||
"baseline": "OUT-baseline-narrowest-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-narrowest-t2",
|
||||
"previous": "OUT-previous-narrowest-t2",
|
||||
"baseline": "OUT-baseline-narrowest-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-narrowest-t3",
|
||||
"previous": "OUT-previous-narrowest-t3",
|
||||
"baseline": "OUT-baseline-narrowest-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-narrowest-t4",
|
||||
"previous": "OUT-previous-narrowest-t4",
|
||||
"baseline": "OUT-baseline-narrowest-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "roomy",
|
||||
"description": "roomy scenario",
|
||||
"softCriteria": [
|
||||
"a good answer for roomy"
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-roomy-t0",
|
||||
"previous": "OUT-previous-roomy-t0",
|
||||
"baseline": "OUT-baseline-roomy-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-roomy-t1",
|
||||
"previous": "OUT-previous-roomy-t1",
|
||||
"baseline": "OUT-baseline-roomy-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-roomy-t2",
|
||||
"previous": "OUT-previous-roomy-t2",
|
||||
"baseline": "OUT-baseline-roomy-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-roomy-t3",
|
||||
"previous": "OUT-previous-roomy-t3",
|
||||
"baseline": "OUT-baseline-roomy-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-roomy-t4",
|
||||
"previous": "OUT-previous-roomy-t4",
|
||||
"baseline": "OUT-baseline-roomy-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,725 +0,0 @@
|
||||
{
|
||||
"skill": "fragile-skill",
|
||||
"generatedAt": "2026-07-24T11:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
"arms": [
|
||||
{
|
||||
"id": "new",
|
||||
"label": "New skill"
|
||||
},
|
||||
{
|
||||
"id": "previous",
|
||||
"label": "Previous version"
|
||||
},
|
||||
{
|
||||
"id": "baseline",
|
||||
"label": "No skill"
|
||||
}
|
||||
],
|
||||
"comparisons": [
|
||||
{
|
||||
"id": "efficacy",
|
||||
"label": "Efficacy",
|
||||
"new": "new",
|
||||
"against": "baseline",
|
||||
"rule": "efficacy"
|
||||
},
|
||||
{
|
||||
"id": "regression",
|
||||
"label": "Regression",
|
||||
"new": "new",
|
||||
"against": "previous",
|
||||
"rule": "regression"
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "both",
|
||||
"description": "both scenario",
|
||||
"softCriteria": [
|
||||
"a good answer for both"
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-both-t0",
|
||||
"previous": "OUT-previous-both-t0",
|
||||
"baseline": "OUT-baseline-both-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-both-t1",
|
||||
"previous": "OUT-previous-both-t1",
|
||||
"baseline": "OUT-baseline-both-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-both-t2",
|
||||
"previous": "OUT-previous-both-t2",
|
||||
"baseline": "OUT-baseline-both-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-both-t3",
|
||||
"previous": "OUT-previous-both-t3",
|
||||
"baseline": "OUT-baseline-both-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "previous",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-both-t4",
|
||||
"previous": "OUT-previous-both-t4",
|
||||
"baseline": "OUT-baseline-both-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "reg-only",
|
||||
"description": "reg-only scenario",
|
||||
"softCriteria": [
|
||||
"a good answer for reg-only"
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-reg-only-t0",
|
||||
"previous": "OUT-previous-reg-only-t0",
|
||||
"baseline": "OUT-baseline-reg-only-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-reg-only-t1",
|
||||
"previous": "OUT-previous-reg-only-t1",
|
||||
"baseline": "OUT-baseline-reg-only-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-reg-only-t2",
|
||||
"previous": "OUT-previous-reg-only-t2",
|
||||
"baseline": "OUT-baseline-reg-only-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-reg-only-t3",
|
||||
"previous": "OUT-previous-reg-only-t3",
|
||||
"baseline": "OUT-baseline-reg-only-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "previous",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-reg-only-t4",
|
||||
"previous": "OUT-previous-reg-only-t4",
|
||||
"baseline": "OUT-baseline-reg-only-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "none",
|
||||
"description": "none scenario",
|
||||
"softCriteria": [
|
||||
"a good answer for none"
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-none-t0",
|
||||
"previous": "OUT-previous-none-t0",
|
||||
"baseline": "OUT-baseline-none-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-none-t1",
|
||||
"previous": "OUT-previous-none-t1",
|
||||
"baseline": "OUT-baseline-none-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-none-t2",
|
||||
"previous": "OUT-previous-none-t2",
|
||||
"baseline": "OUT-baseline-none-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-none-t3",
|
||||
"previous": "OUT-previous-none-t3",
|
||||
"baseline": "OUT-baseline-none-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
},
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "fixture"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-none-t4",
|
||||
"previous": "OUT-previous-none-t4",
|
||||
"baseline": "OUT-baseline-none-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 500,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 300,
|
||||
"turns": 3
|
||||
},
|
||||
"previous": {
|
||||
"input": 900,
|
||||
"output": 450,
|
||||
"cacheCreation": 180,
|
||||
"cacheRead": 270,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 600,
|
||||
"output": 300,
|
||||
"cacheCreation": 0,
|
||||
"cacheRead": 0,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,954 +0,0 @@
|
||||
{
|
||||
"skill": "sample-skill",
|
||||
"generatedAt": "2026-07-24T12:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
"arms": [
|
||||
{
|
||||
"id": "new",
|
||||
"label": "New skill"
|
||||
},
|
||||
{
|
||||
"id": "previous",
|
||||
"label": "Previous version"
|
||||
},
|
||||
{
|
||||
"id": "baseline",
|
||||
"label": "No skill"
|
||||
}
|
||||
],
|
||||
"comparisons": [
|
||||
{
|
||||
"id": "efficacy",
|
||||
"label": "Efficacy",
|
||||
"new": "new",
|
||||
"against": "baseline",
|
||||
"rule": "efficacy"
|
||||
},
|
||||
{
|
||||
"id": "regression",
|
||||
"label": "Regression",
|
||||
"new": "new",
|
||||
"against": "previous",
|
||||
"rule": "regression"
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "clean-both",
|
||||
"description": "New beats baseline and holds steady versus the previous version.",
|
||||
"softCriteria": [
|
||||
"The answer is grounded in the fixture."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t0",
|
||||
"previous": "OUT-previous-clean-both-t0",
|
||||
"baseline": "OUT-baseline-clean-both-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t1",
|
||||
"previous": "OUT-previous-clean-both-t1",
|
||||
"baseline": "OUT-baseline-clean-both-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t2",
|
||||
"previous": "OUT-previous-clean-both-t2",
|
||||
"baseline": "OUT-baseline-clean-both-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t3",
|
||||
"previous": "OUT-previous-clean-both-t3",
|
||||
"baseline": "OUT-baseline-clean-both-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-both-t4",
|
||||
"previous": "OUT-previous-clean-both-t4",
|
||||
"baseline": "OUT-baseline-clean-both-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "regressed-still-valuable",
|
||||
"description": "New still beats baseline but has degraded from the previous version.",
|
||||
"softCriteria": [
|
||||
"The answer resolves the user's request."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "previous",
|
||||
"rationale": "judge: regression previous"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t0",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t0",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "previous",
|
||||
"rationale": "judge: regression previous"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t1",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t1",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t2",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t2",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t3",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t3",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regressed-still-valuable-t4",
|
||||
"previous": "OUT-previous-regressed-still-valuable-t4",
|
||||
"baseline": "OUT-baseline-regressed-still-valuable-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "dead-weight",
|
||||
"description": "New does not beat baseline, but the edit did not make it worse.",
|
||||
"softCriteria": [
|
||||
"The answer is correct."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t0",
|
||||
"previous": "OUT-previous-dead-weight-t0",
|
||||
"baseline": "OUT-baseline-dead-weight-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t1",
|
||||
"previous": "OUT-previous-dead-weight-t1",
|
||||
"baseline": "OUT-baseline-dead-weight-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge: efficacy baseline"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t2",
|
||||
"previous": "OUT-previous-dead-weight-t2",
|
||||
"baseline": "OUT-baseline-dead-weight-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: regression new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t3",
|
||||
"previous": "OUT-previous-dead-weight-t3",
|
||||
"baseline": "OUT-baseline-dead-weight-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-dead-weight-t4",
|
||||
"previous": "OUT-previous-dead-weight-t4",
|
||||
"baseline": "OUT-baseline-dead-weight-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hard-gate-fail",
|
||||
"description": "New sweeps both head-to-heads but violates a hard assertion.",
|
||||
"softCriteria": [
|
||||
"The output is well-formed."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t0",
|
||||
"previous": "OUT-previous-hard-gate-fail-t0",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t1",
|
||||
"previous": "OUT-previous-hard-gate-fail-t1",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t2",
|
||||
"previous": "OUT-previous-hard-gate-fail-t2",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t3",
|
||||
"previous": "OUT-previous-hard-gate-fail-t3",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
},
|
||||
"regression": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: regression tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t4",
|
||||
"previous": "OUT-previous-hard-gate-fail-t4",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
},
|
||||
"previous": {
|
||||
"input": 800,
|
||||
"output": 1600,
|
||||
"cacheCreation": 300,
|
||||
"cacheRead": 6000,
|
||||
"turns": 3
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -1,515 +0,0 @@
|
||||
{
|
||||
"skill": "sample-skill-2arm",
|
||||
"generatedAt": "2026-07-24T12:00:00Z",
|
||||
"temperature": 1.0,
|
||||
"trialsPerCase": 5,
|
||||
"arms": [
|
||||
{
|
||||
"id": "new",
|
||||
"label": "New skill"
|
||||
},
|
||||
{
|
||||
"id": "baseline",
|
||||
"label": "No skill"
|
||||
}
|
||||
],
|
||||
"comparisons": [
|
||||
{
|
||||
"id": "efficacy",
|
||||
"label": "Efficacy",
|
||||
"new": "new",
|
||||
"against": "baseline",
|
||||
"rule": "efficacy"
|
||||
}
|
||||
],
|
||||
"cases": [
|
||||
{
|
||||
"name": "clean-pass",
|
||||
"description": "New skill reliably beats the no-skill baseline.",
|
||||
"softCriteria": [
|
||||
"The answer is grounded in the fixture.",
|
||||
"The answer follows the skill's format."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t0",
|
||||
"baseline": "OUT-baseline-clean-pass-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t1",
|
||||
"baseline": "OUT-baseline-clean-pass-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t2",
|
||||
"baseline": "OUT-baseline-clean-pass-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t3",
|
||||
"baseline": "OUT-baseline-clean-pass-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-clean-pass-t4",
|
||||
"baseline": "OUT-baseline-clean-pass-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "regresses-baseline",
|
||||
"description": "New skill does not reliably beat the baseline.",
|
||||
"softCriteria": [
|
||||
"The answer resolves the user's request."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t0",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t1",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "tie",
|
||||
"rationale": "judge: efficacy tie"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t2",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge: efficacy baseline"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t3",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "baseline",
|
||||
"rationale": "judge: efficacy baseline"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-regresses-baseline-t4",
|
||||
"baseline": "OUT-baseline-regresses-baseline-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
}
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hard-gate-fail",
|
||||
"description": "Output wins the head-to-head but violates a hard assertion.",
|
||||
"softCriteria": [
|
||||
"The output is correct."
|
||||
],
|
||||
"trials": [
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t0",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t0"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t1",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t1"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t2",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t2"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": false
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t3",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t3"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
},
|
||||
{
|
||||
"comparisons": {
|
||||
"efficacy": {
|
||||
"winner": "new",
|
||||
"rationale": "judge: efficacy new"
|
||||
}
|
||||
},
|
||||
"outputs": {
|
||||
"new": "OUT-new-hard-gate-fail-t4",
|
||||
"baseline": "OUT-baseline-hard-gate-fail-t4"
|
||||
},
|
||||
"usage": {
|
||||
"new": {
|
||||
"input": 1000,
|
||||
"output": 2000,
|
||||
"cacheCreation": 400,
|
||||
"cacheRead": 8000,
|
||||
"turns": 3
|
||||
},
|
||||
"baseline": {
|
||||
"input": 500,
|
||||
"output": 1000,
|
||||
"cacheCreation": 200,
|
||||
"cacheRead": 4000,
|
||||
"turns": 2
|
||||
}
|
||||
},
|
||||
"hard": {
|
||||
"ran": true,
|
||||
"pass": true
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
Reference in New Issue
Block a user