{
  "skill": "fragile-skill",
  "generatedAt": "2026-07-24T11:00:00Z",
  "temperature": 1.0,
  "trialsPerCase": 5,
  "arms": [
    {
      "id": "new",
      "label": "New skill"
    },
    {
      "id": "previous",
      "label": "Previous version"
    },
    {
      "id": "baseline",
      "label": "No skill"
    }
  ],
  "comparisons": [
    {
      "id": "efficacy",
      "label": "Efficacy",
      "new": "new",
      "against": "baseline",
      "rule": "efficacy"
    },
    {
      "id": "regression",
      "label": "Regression",
      "new": "new",
      "against": "previous",
      "rule": "regression"
    }
  ],
  "cases": [
    {
      "name": "both",
      "description": "both scenario",
      "softCriteria": [
        "a good answer for both"
      ],
      "trials": [
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "new",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-both-t0",
            "previous": "OUT-previous-both-t0",
            "baseline": "OUT-baseline-both-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-both-t1",
            "previous": "OUT-previous-both-t1",
            "baseline": "OUT-baseline-both-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-both-t2",
            "previous": "OUT-previous-both-t2",
            "baseline": "OUT-baseline-both-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-both-t3",
            "previous": "OUT-previous-both-t3",
            "baseline": "OUT-baseline-both-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "previous",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-both-t4",
            "previous": "OUT-previous-both-t4",
            "baseline": "OUT-baseline-both-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        }
      ]
    },
    {
      "name": "reg-only",
      "description": "reg-only scenario",
      "softCriteria": [
        "a good answer for reg-only"
      ],
      "trials": [
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-reg-only-t0",
            "previous": "OUT-previous-reg-only-t0",
            "baseline": "OUT-baseline-reg-only-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-reg-only-t1",
            "previous": "OUT-previous-reg-only-t1",
            "baseline": "OUT-baseline-reg-only-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-reg-only-t2",
            "previous": "OUT-previous-reg-only-t2",
            "baseline": "OUT-baseline-reg-only-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-reg-only-t3",
            "previous": "OUT-previous-reg-only-t3",
            "baseline": "OUT-baseline-reg-only-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "previous",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-reg-only-t4",
            "previous": "OUT-previous-reg-only-t4",
            "baseline": "OUT-baseline-reg-only-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        }
      ]
    },
    {
      "name": "none",
      "description": "none scenario",
      "softCriteria": [
        "a good answer for none"
      ],
      "trials": [
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-none-t0",
            "previous": "OUT-previous-none-t0",
            "baseline": "OUT-baseline-none-t0"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-none-t1",
            "previous": "OUT-previous-none-t1",
            "baseline": "OUT-baseline-none-t1"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-none-t2",
            "previous": "OUT-previous-none-t2",
            "baseline": "OUT-baseline-none-t2"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-none-t3",
            "previous": "OUT-previous-none-t3",
            "baseline": "OUT-baseline-none-t3"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        },
        {
          "hard": {
            "ran": true,
            "pass": true
          },
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "fixture"
            },
            "regression": {
              "winner": "tie",
              "rationale": "fixture"
            }
          },
          "outputs": {
            "new": "OUT-new-none-t4",
            "previous": "OUT-previous-none-t4",
            "baseline": "OUT-baseline-none-t4"
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 500,
              "cacheCreation": 200,
              "cacheRead": 300,
              "turns": 3
            },
            "previous": {
              "input": 900,
              "output": 450,
              "cacheCreation": 180,
              "cacheRead": 270,
              "turns": 3
            },
            "baseline": {
              "input": 600,
              "output": 300,
              "cacheCreation": 0,
              "cacheRead": 0,
              "turns": 2
            }
          }
        }
      ]
    }
  ]
}