# Feeds the committed run-bundle fixtures through the benchmark skill's
# deterministic core and asserts the per-arm metrics, the per-case and
# skill-level Efficacy and Regression verdicts, and the rendered report — for
# both the three-arm shape and the degenerate two-arm "no previous version"
# shape.
# No LLM runs.
{
  pkgs,
}:
let
  core = ../skills/benchmark-skill/core/benchmark_core.py;
  threeArm = ./fixtures/benchmark/three-arm-bundle.json;
  twoArm = ./fixtures/benchmark/two-arm-bundle.json;
in
pkgs.runCommandLocal "benchmark-core-check"
  {
    nativeBuildInputs = [ pkgs.python3 ];
    inherit core threeArm twoArm;
  }
  ''
    fail() { echo "FAIL: $1" >&2; exit 1; }

    # --- three-arm run --------------------------------------------------------
    echo "the core scores a three-arm run and renders its report"
    python3 "$core" "$threeArm" --json three.json --html three.html \
      || fail "the core exited non-zero on the three-arm bundle"

    echo "the three-arm results model carries both verdicts, net margins, and metrics"
    python3 - three.json <<'PY' || fail "a three-arm results-model assertion failed"
    import json, sys
    r = json.load(open(sys.argv[1]))

    assert r["armShape"] == "three-arm", r["armShape"]
    # Efficacy is red: dead-weight fails the wins floor and hard-gate-fail trips
    # the gate. Regression is red: regressed-still-valuable loses twice to the
    # previous version.
    assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
    assert r["regressionVerdict"] == "red", r["regressionVerdict"]
    assert r["efficacyNetMargin"] == 12, r["efficacyNetMargin"]
    assert r["regressionNetMargin"] == 3, r["regressionNetMargin"]

    new = r["armMetrics"]["new"]
    assert new["turns"] == 60, new["turns"]
    assert new["rawTokens"] == 228000, new["rawTokens"]
    assert new["costEquivalentTokens"] == 246000, new["costEquivalentTokens"]
    assert abs(new["imputedCost"] - 1.23) < 1e-9, new["imputedCost"]

    base = r["armMetrics"]["baseline"]
    assert base["turns"] == 40, base["turns"]
    assert base["rawTokens"] == 114000, base["rawTokens"]
    assert base["costEquivalentTokens"] == 123000, base["costEquivalentTokens"]

    prev = r["armMetrics"]["previous"]
    assert prev["turns"] == 60, prev["turns"]
    assert prev["rawTokens"] == 174000, prev["rawTokens"]
    assert prev["costEquivalentTokens"] == 195500, prev["costEquivalentTokens"]
    assert abs(prev["imputedCost"] - 0.9775) < 1e-9, prev["imputedCost"]

    # Cost-table rows render in arm order: new-skill, previous-version, no-skill.
    assert [a["id"] for a in r["arms"]] == ["new", "previous", "baseline"], \
        [a["id"] for a in r["arms"]]

    cases = {c["name"]: c for c in r["cases"]}
    # Authored order is preserved, never reshuffled by verdict.
    assert [c["name"] for c in r["cases"]] == [
        "clean-both", "regressed-still-valuable", "dead-weight", "hard-gate-fail"
    ]

    a = cases["clean-both"]
    assert a["efficacyPassed"] is True
    assert a["regressionPassed"] is True
    assert a["comparisons"]["efficacy"]["trials"] == ["WIN", "WIN", "WIN", "WIN", "TIE"]
    assert a["comparisons"]["regression"]["trials"] == ["TIE", "TIE", "WIN", "TIE", "TIE"]

    # Efficacy green but regression red — the exact crossing this feature exists
    # to catch: still beats no-skill, yet degraded from the released version.
    b = cases["regressed-still-valuable"]
    assert b["efficacyPassed"] is True
    assert b["regressionPassed"] is False
    assert b["comparisons"]["regression"]["losses"] == 2
    assert b["comparisons"]["regression"]["flaggedLosses"] == [0, 1]

    # Efficacy red but regression green — already dead weight, but the edit did
    # not make it worse.
    c = cases["dead-weight"]
    assert c["efficacyPassed"] is False
    assert c["regressionPassed"] is True
    assert c["comparisons"]["efficacy"]["wins"] == 0
    assert c["comparisons"]["efficacy"]["losses"] == 1
    assert c["comparisons"]["efficacy"]["flaggedLosses"] == [2]
    assert c["comparisons"]["regression"]["wins"] == 3

    # The hard gate fails efficacy outright but does not gate regression, which
    # is judged purely on losses.
    d = cases["hard-gate-fail"]
    assert d["hardFailed"] is True
    assert d["efficacyPassed"] is False
    assert d["regressionPassed"] is True
    assert d["comparisons"]["efficacy"]["wins"] == 5
    print("three-arm results-model assertions passed")
    PY

    echo "the three-arm report shows both badges, the three-row table, and evidence"
    grep -q '<!doctype html>' three.html || fail "three-arm report is not self-contained"
    grep -q 'Efficacy: RED' three.html || fail "three-arm report is missing the red Efficacy badge"
    grep -q 'Regression: RED' three.html || fail "three-arm report is missing the red Regression badge"
    grep -qi 'regress' three.html || fail "three-arm report is missing the verdict headline"
    grep -q 'Cost-equiv tokens' three.html || fail "three-arm report is missing the cost table"
    grep -q 'Previous version' three.html || fail "three-arm report is missing the previous-version row"
    grep -q 'shared-context cache' three.html || fail "three-arm report is missing the cache footnote"
    grep -q 'new-vs-no-skill' three.html || fail "three-arm footnote omits the new-vs-no-skill ratio"
    grep -q 'new-vs-old' three.html || fail "three-arm footnote omits the new-vs-old ratio"
    for name in clean-both regressed-still-valuable dead-weight hard-gate-fail; do
      grep -q "$name" three.html || fail "three-arm report omits case $name"
    done
    grep -q 'class="cell LOSS"' three.html || fail "three-arm report is missing a LOSS cell"
    grep -q 'Hard assertion failed' three.html || fail "three-arm report does not flag the hard failure"
    grep -q ' open>' three.html || fail "three-arm report does not auto-expand a failing case"
    grep -q 'class="cmps"' three.html || fail "three-arm report does not lay comparisons side by side"
    # Evidence for the failed regression comparison: new vs previous on trial 1.
    grep -q 'OUT-new-regressed-still-valuable-t0' three.html || fail "missing new evidence for the regression loss"
    grep -q 'OUT-previous-regressed-still-valuable-t0' three.html || fail "missing previous evidence for the regression loss"
    # Evidence for the failed efficacy comparison: new vs baseline on trial 3.
    grep -q 'OUT-new-dead-weight-t2' three.html || fail "missing new evidence for the efficacy loss"
    grep -q 'OUT-baseline-dead-weight-t2' three.html || fail "missing baseline evidence for the efficacy loss"
    # A clean, collapsed case emits no losing-trial evidence.
    if grep -q 'OUT-new-clean-both' three.html; then fail "a clean case leaked losing-trial evidence"; fi

    # --- two-arm run ----------------------------------------------------------
    echo "the core scores the degenerate two-arm run"
    python3 "$core" "$twoArm" --json two.json --html two.html \
      || fail "the core exited non-zero on the two-arm bundle"

    echo "the two-arm results model reads regression as not-applicable"
    python3 - two.json <<'PY' || fail "a two-arm results-model assertion failed"
    import json, sys
    r = json.load(open(sys.argv[1]))

    assert r["armShape"] == "two-arm", r["armShape"]
    assert r["efficacyVerdict"] == "red", r["efficacyVerdict"]
    assert r["regressionVerdict"] == "not-applicable", r["regressionVerdict"]
    assert r["regressionNetMargin"] is None, r["regressionNetMargin"]
    assert r["efficacyNetMargin"] == 9, r["efficacyNetMargin"]

    new = r["armMetrics"]["new"]
    assert new["turns"] == 45, new["turns"]
    assert new["rawTokens"] == 171000, new["rawTokens"]
    assert new["costEquivalentTokens"] == 184500, new["costEquivalentTokens"]
    assert abs(new["imputedCost"] - 0.9225) < 1e-9, new["imputedCost"]

    base = r["armMetrics"]["baseline"]
    assert base["turns"] == 30, base["turns"]
    assert base["rawTokens"] == 85500, base["rawTokens"]
    assert base["costEquivalentTokens"] == 92250, base["costEquivalentTokens"]

    assert "previous" not in r["armMetrics"], "two-arm run has no previous arm"

    assert [c["name"] for c in r["cases"]] == [
        "clean-pass", "regresses-baseline", "hard-gate-fail"
    ]
    for c in r["cases"]:
        assert c["regressionPassed"] is None, (c["name"], c["regressionPassed"])

    b = {c["name"]: c for c in r["cases"]}["regresses-baseline"]
    assert b["efficacyPassed"] is False
    assert b["comparisons"]["efficacy"]["losses"] == 2
    assert b["comparisons"]["efficacy"]["flaggedLosses"] == [3, 4]

    h = {c["name"]: c for c in r["cases"]}["hard-gate-fail"]
    assert h["hardFailed"] is True
    print("two-arm results-model assertions passed")
    PY

    echo "the two-arm report reads Regression as not-applicable and drops the previous row"
    grep -q 'Efficacy: RED' two.html || fail "two-arm report is missing the Efficacy badge"
    grep -q 'Regression: N/A' two.html || fail "two-arm report does not read Regression as not-applicable"
    grep -q 'new-vs-no-skill' two.html || fail "two-arm footnote omits the new-vs-no-skill ratio"
    if grep -q 'new-vs-old' two.html; then fail "two-arm footnote names a new-vs-old ratio that does not apply"; fi
    if grep -q 'Previous version' two.html; then fail "two-arm report shows a previous-version row"; fi
    grep -q 'OUT-new-regresses-baseline-t3' two.html || fail "missing new evidence for the two-arm efficacy loss"
    grep -q 'OUT-baseline-regresses-baseline-t3' two.html || fail "missing baseline evidence for the two-arm efficacy loss"

    touch "$out"
  ''