{
  "skill": "sample-skill",
  "generatedAt": "2026-07-24T12:00:00Z",
  "temperature": 1.0,
  "trialsPerCase": 5,
  "arms": [
    {
      "id": "new",
      "label": "New skill"
    },
    {
      "id": "previous",
      "label": "Previous version"
    },
    {
      "id": "baseline",
      "label": "No skill"
    }
  ],
  "comparisons": [
    {
      "id": "efficacy",
      "label": "Efficacy",
      "new": "new",
      "against": "baseline",
      "rule": "efficacy"
    },
    {
      "id": "regression",
      "label": "Regression",
      "new": "new",
      "against": "previous",
      "rule": "regression"
    }
  ],
  "cases": [
    {
      "name": "clean-both",
      "description": "New beats baseline and holds steady versus the previous version.",
      "softCriteria": [
        "The answer is grounded in the fixture."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-clean-both-t0",
            "previous": "OUT-previous-clean-both-t0",
            "baseline": "OUT-baseline-clean-both-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-clean-both-t1",
            "previous": "OUT-previous-clean-both-t1",
            "baseline": "OUT-baseline-clean-both-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "new",
              "rationale": "judge: regression new"
            }
          },
          "outputs": {
            "new": "OUT-new-clean-both-t2",
            "previous": "OUT-previous-clean-both-t2",
            "baseline": "OUT-baseline-clean-both-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-clean-both-t3",
            "previous": "OUT-previous-clean-both-t3",
            "baseline": "OUT-baseline-clean-both-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge: efficacy tie"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-clean-both-t4",
            "previous": "OUT-previous-clean-both-t4",
            "baseline": "OUT-baseline-clean-both-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        }
      ]
    },
    {
      "name": "regressed-still-valuable",
      "description": "New still beats baseline but has degraded from the previous version.",
      "softCriteria": [
        "The answer resolves the user's request."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "previous",
              "rationale": "judge: regression previous"
            }
          },
          "outputs": {
            "new": "OUT-new-regressed-still-valuable-t0",
            "previous": "OUT-previous-regressed-still-valuable-t0",
            "baseline": "OUT-baseline-regressed-still-valuable-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "previous",
              "rationale": "judge: regression previous"
            }
          },
          "outputs": {
            "new": "OUT-new-regressed-still-valuable-t1",
            "previous": "OUT-previous-regressed-still-valuable-t1",
            "baseline": "OUT-baseline-regressed-still-valuable-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-regressed-still-valuable-t2",
            "previous": "OUT-previous-regressed-still-valuable-t2",
            "baseline": "OUT-baseline-regressed-still-valuable-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge: efficacy tie"
            },
            "regression": {
              "winner": "new",
              "rationale": "judge: regression new"
            }
          },
          "outputs": {
            "new": "OUT-new-regressed-still-valuable-t3",
            "previous": "OUT-previous-regressed-still-valuable-t3",
            "baseline": "OUT-baseline-regressed-still-valuable-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-regressed-still-valuable-t4",
            "previous": "OUT-previous-regressed-still-valuable-t4",
            "baseline": "OUT-baseline-regressed-still-valuable-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        }
      ]
    },
    {
      "name": "dead-weight",
      "description": "New does not beat baseline, but the edit did not make it worse.",
      "softCriteria": [
        "The answer is correct."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge: efficacy tie"
            },
            "regression": {
              "winner": "new",
              "rationale": "judge: regression new"
            }
          },
          "outputs": {
            "new": "OUT-new-dead-weight-t0",
            "previous": "OUT-previous-dead-weight-t0",
            "baseline": "OUT-baseline-dead-weight-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge: efficacy tie"
            },
            "regression": {
              "winner": "new",
              "rationale": "judge: regression new"
            }
          },
          "outputs": {
            "new": "OUT-new-dead-weight-t1",
            "previous": "OUT-previous-dead-weight-t1",
            "baseline": "OUT-baseline-dead-weight-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "baseline",
              "rationale": "judge: efficacy baseline"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-dead-weight-t2",
            "previous": "OUT-previous-dead-weight-t2",
            "baseline": "OUT-baseline-dead-weight-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge: efficacy tie"
            },
            "regression": {
              "winner": "new",
              "rationale": "judge: regression new"
            }
          },
          "outputs": {
            "new": "OUT-new-dead-weight-t3",
            "previous": "OUT-previous-dead-weight-t3",
            "baseline": "OUT-baseline-dead-weight-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge: efficacy tie"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-dead-weight-t4",
            "previous": "OUT-previous-dead-weight-t4",
            "baseline": "OUT-baseline-dead-weight-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        }
      ]
    },
    {
      "name": "hard-gate-fail",
      "description": "New sweeps both head-to-heads but violates a hard assertion.",
      "softCriteria": [
        "The output is well-formed."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-hard-gate-fail-t0",
            "previous": "OUT-previous-hard-gate-fail-t0",
            "baseline": "OUT-baseline-hard-gate-fail-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-hard-gate-fail-t1",
            "previous": "OUT-previous-hard-gate-fail-t1",
            "baseline": "OUT-baseline-hard-gate-fail-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-hard-gate-fail-t2",
            "previous": "OUT-previous-hard-gate-fail-t2",
            "baseline": "OUT-baseline-hard-gate-fail-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": false
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-hard-gate-fail-t3",
            "previous": "OUT-previous-hard-gate-fail-t3",
            "baseline": "OUT-baseline-hard-gate-fail-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge: efficacy new"
            },
            "regression": {
              "winner": "tie",
              "rationale": "judge: regression tie"
            }
          },
          "outputs": {
            "new": "OUT-new-hard-gate-fail-t4",
            "previous": "OUT-previous-hard-gate-fail-t4",
            "baseline": "OUT-baseline-hard-gate-fail-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            },
            "previous": {
              "input": 800,
              "output": 1600,
              "cacheCreation": 300,
              "cacheRead": 6000,
              "turns": 3
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        }
      ]
    }
  ]
}