"""Tests for benchmark aggregation."""

import json
from pathlib import Path

from skill_eval_runner.benchmark import compute_benchmark, stats
from skill_eval_runner.suite import EvalCase


def test_stats_empty() -> None:
    """Empty input yields zeroed stats."""
    assert stats([]) == {"mean": 0.0, "stddev": 0.0}


def test_stats_mean_and_stddev() -> None:
    """Mean and population stddev are computed."""
    assert stats([1.0, 3.0]) == {"mean": 2.0, "stddev": 1.0}


def _write_outputs(
    iteration: Path, case: EvalCase, config: str, pass_rate: float, tokens: int
) -> None:
    """Write a case/config's grading.json + timing.json under an iteration dir."""
    out = iteration / case.dir_name() / config
    out.mkdir(parents=True)
    (out / "grading.json").write_text(json.dumps({"summary": {"pass_rate": pass_rate}}))
    (out / "timing.json").write_text(
        json.dumps({"duration_ms": 1000, "total_tokens": tokens, "cost_usd": 0.01})
    )


def test_compute_benchmark_delta(tmp_path: Path) -> None:
    """The headline delta is the difference of mean pass rates across configs."""
    case = EvalCase(name="c", id="c", prompt="p")
    _write_outputs(tmp_path, case, "with_skill", 1.0, 100)
    _write_outputs(tmp_path, case, "without_skill", 0.5, 80)

    summary = compute_benchmark(tmp_path, [case])["run_summary"]
    assert summary["with_skill"]["pass_rate"]["mean"] == 1.0
    assert summary["without_skill"]["pass_rate"]["mean"] == 0.5
    assert summary["delta"]["pass_rate"] == 0.5
    assert summary["delta"]["cost_usd"] == 0.0


def test_cost_prefers_total_cost_usd(tmp_path: Path) -> None:
    """Aggregated cost uses total_cost_usd (agent + grader) over the agent-only cost_usd."""
    case = EvalCase(name="c", id="c", prompt="p")
    out = tmp_path / case.dir_name() / "with_skill"
    out.mkdir(parents=True)
    (out / "grading.json").write_text(json.dumps({"summary": {"pass_rate": 1.0}}))
    (out / "timing.json").write_text(
        json.dumps(
            {
                "duration_ms": 1000,
                "total_tokens": 100,
                "cost_usd": 0.01,  # agent only
                "total_cost_usd": 0.013,  # agent + grader — should win
            }
        )
    )
    summary = compute_benchmark(tmp_path, [case])["run_summary"]
    assert summary["with_skill"]["cost_usd"]["mean"] == 0.013


def test_cost_delta_null_when_a_config_cost_unknown(tmp_path: Path) -> None:
    """If either config's cost is unknown, delta.cost_usd is null, not a misleading 0.0."""
    case = EvalCase(name="c", id="c", prompt="p")
    _write_outputs(tmp_path, case, "with_skill", 1.0, 100)  # cost_usd known (0.01)

    # without_skill: pass-rate/tokens known, but cost unknown (model not in pricing table).
    out = tmp_path / case.dir_name() / "without_skill"
    out.mkdir(parents=True)
    (out / "grading.json").write_text(json.dumps({"summary": {"pass_rate": 0.5}}))
    (out / "timing.json").write_text(
        json.dumps({"duration_ms": 1000, "total_tokens": 80, "cost_usd": None})
    )

    summary = compute_benchmark(tmp_path, [case])["run_summary"]
    assert summary["delta"]["cost_usd"] is None