{
  "skill": "sample-skill",
  "generatedAt": "2026-07-24T12:00:00Z",
  "temperature": 1.0,
  "trialsPerCase": 5,
  "arms": [
    {
      "id": "new",
      "label": "New skill"
    },
    {
      "id": "baseline",
      "label": "No skill"
    }
  ],
  "comparisons": [
    {
      "id": "efficacy",
      "label": "Efficacy",
      "new": "new",
      "against": "baseline",
      "rule": "efficacy"
    }
  ],
  "cases": [
    {
      "name": "clean-pass",
      "description": "New skill reliably beats the no-skill baseline.",
      "softCriteria": [
        "The answer is grounded in the fixture.",
        "The answer follows the skill's format."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge preferred tie"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        }
      ]
    },
    {
      "name": "regresses-baseline",
      "description": "New skill does not reliably beat the baseline.",
      "softCriteria": [
        "The answer resolves the user's request."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "tie",
              "rationale": "judge preferred tie"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "baseline",
              "rationale": "judge preferred baseline"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "baseline",
              "rationale": "judge preferred baseline"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          }
        }
      ]
    },
    {
      "name": "hard-gate-fail",
      "description": "Output wins the head-to-head but violates a hard assertion.",
      "softCriteria": [
        "The output is correct."
      ],
      "trials": [
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": false
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        },
        {
          "comparisons": {
            "efficacy": {
              "winner": "new",
              "rationale": "judge preferred new"
            }
          },
          "usage": {
            "new": {
              "input": 1000,
              "output": 2000,
              "cacheCreation": 400,
              "cacheRead": 8000,
              "turns": 3
            },
            "baseline": {
              "input": 500,
              "output": 1000,
              "cacheCreation": 200,
              "cacheRead": 4000,
              "turns": 2
            }
          },
          "hard": {
            "ran": true,
            "pass": true
          }
        }
      ]
    }
  ]
}