"""Benchmark aggregation: roll up an iteration's per-case outputs into stats.

Reads the ``grading.json`` / ``timing.json`` each case wrote under both configs and
produces mean/stddev for pass-rate, time, tokens, and cost — plus the headline
``with_skill - without_skill`` delta.
"""

import json
import math
from pathlib import Path
from typing import Any

from skill_eval_runner.suite import EvalCase

CONFIGS = ("with_skill", "without_skill")


def stats(values: list[float]) -> dict[str, float]:
    """Return the mean and population stddev of ``values`` (zeros when empty)."""
    if not values:
        return {"mean": 0.0, "stddev": 0.0}
    mean = sum(values) / len(values)
    variance = sum((v - mean) ** 2 for v in values) / len(values)
    return {"mean": round(mean, 3), "stddev": round(math.sqrt(variance), 3)}


def _config_summary(
    iteration_dir: Path, config: str, cases: list[EvalCase]
) -> dict[str, Any]:
    """Aggregate one config's outputs across all cases."""
    pass_rates: list[float] = []
    times: list[float] = []
    tokens: list[float] = []
    costs: list[float] = []
    for case in cases:
        case_dir = iteration_dir / case.dir_name() / config
        grading_path = case_dir / "grading.json"
        timing_path = case_dir / "timing.json"
        if grading_path.exists():
            grading = json.loads(grading_path.read_text())
            pass_rates.append(grading["summary"]["pass_rate"])
        if timing_path.exists():
            timing = json.loads(timing_path.read_text())
            times.append(timing["duration_ms"] / 1000)
            tokens.append(timing["total_tokens"])
            # Prefer the agent + grader total; fall back to agent-only cost for
            # timing.json written before grader cost was tracked.
            cost = timing.get("total_cost_usd")
            if cost is None:
                cost = timing.get("cost_usd")
            if cost is not None:
                costs.append(cost)
    return {
        "pass_rate": stats(pass_rates),
        "time_seconds": stats(times),
        "tokens": stats(tokens),
        "cost_usd": stats(costs) if costs else None,
    }


def _mean(summary: dict[str, Any], metric: str) -> float:
    """Return the mean of ``metric`` from a config summary (0.0 if absent)."""
    entry = summary.get(metric) or {}
    return float(entry.get("mean", 0.0))


def _cost_delta(
    with_skill: dict[str, Any], without_skill: dict[str, Any]
) -> float | None:
    """Return the with−without cost delta, or None if either config's cost is unknown.

    Cost is None when a model isn't in the pricing table; treating that as 0.0 would
    report a misleading delta, so the delta is null unless both sides are known.
    """
    with_cost = with_skill.get("cost_usd")
    without_cost = without_skill.get("cost_usd")
    if with_cost is None or without_cost is None:
        return None
    return round(float(with_cost["mean"]) - float(without_cost["mean"]), 6)


def compute_benchmark(iteration_dir: Path, cases: list[EvalCase]) -> dict[str, Any]:
    """Aggregate an iteration's outputs into per-config stats and the with/without delta."""
    summary: dict[str, Any] = {
        config: _config_summary(iteration_dir, config, cases) for config in CONFIGS
    }
    with_skill = summary["with_skill"]
    without_skill = summary["without_skill"]
    summary["delta"] = {
        "pass_rate": round(
            _mean(with_skill, "pass_rate") - _mean(without_skill, "pass_rate"), 3
        ),
        "time_seconds": round(
            _mean(with_skill, "time_seconds") - _mean(without_skill, "time_seconds"), 3
        ),
        "tokens": round(
            _mean(with_skill, "tokens") - _mean(without_skill, "tokens"), 3
        ),
        "cost_usd": _cost_delta(with_skill, without_skill),
    }
    return {"run_summary": summary}