"""Command-line interface and suite orchestration.

Subcommands encode the workflow people use: ``iterate`` (small model, skill-only)
to tune cheaply, ``validate`` (large model, with baseline) to confirm a real lift,
``run`` for the raw flags, and ``grade`` to re-grade existing outputs.
"""

import argparse
import json
import logging
import os
import sys
from pathlib import Path
from typing import Any

from skill_eval_runner.benchmark import compute_benchmark
from skill_eval_runner.config import EvalSuiteConfig, load_config
from skill_eval_runner.providers import Provider, get_provider, provider_names
from skill_eval_runner.suite import load_suite
from skill_eval_runner.workspace import grade_only, resolve_iteration, run_and_save

COMMANDS = ("iterate", "validate", "run", "grade")


def run_suite(
    config: EvalSuiteConfig,
    provider: Provider,
    model_name: str,
    grader_model_name: str,
    *,
    case_id: str | None = None,
    no_baseline: bool = False,
    grade_only_mode: bool = False,
    iter_force: int | None = None,
) -> dict[str, Any]:
    """Run (or re-grade) the suite end to end and return the benchmark dict."""
    suite = load_suite(config.suite)
    all_cases = suite.evals
    cases = all_cases
    if case_id:
        cases = [c for c in all_cases if c.case_id() == case_id]
        if not cases:
            raise SystemExit(f"No eval case with id '{case_id}'")

    iteration_dir = resolve_iteration(config.workspace, iter_force)
    print(f"Iteration: {iteration_dir.name}")
    print(f"Provider: {provider.name}   Model: {model_name}")
    if grader_model_name != model_name:
        print(f"Grader model: {grader_model_name}")
    print()

    model = provider.make_model(model_name)
    # Reuse the agent model instance when the grader is the same model (the default).
    grader_model = (
        model
        if grader_model_name == model_name
        else provider.make_model(grader_model_name)
    )
    configs = ["with_skill"] + ([] if no_baseline else ["without_skill"])

    total_cost: float | None = None
    for case in cases:
        print(f"=== {case.case_id()} — {case.name} ===")
        for config_name in configs:
            if grade_only_mode:
                cost = grade_only(
                    case,
                    config_name,
                    iteration_dir,
                    grader_model,
                    grader_model_name,
                    config,
                )
            else:
                cost = run_and_save(
                    case,
                    config_name,
                    iteration_dir,
                    model,
                    model_name,
                    grader_model,
                    grader_model_name,
                    config,
                )
            if cost is not None:
                total_cost = (total_cost or 0.0) + cost
        print()

    benchmark = compute_benchmark(iteration_dir, all_cases)
    (iteration_dir / "benchmark.json").write_text(json.dumps(benchmark, indent=2))
    print(f"Benchmark saved to {iteration_dir / 'benchmark.json'}")

    delta = benchmark["run_summary"].get("delta", {})
    pass_rate_delta = delta.get("pass_rate")
    if isinstance(pass_rate_delta, float):
        print(f"Pass rate delta (with - without skill): {pass_rate_delta:+.1%}")

    if total_cost is not None:
        print(f"Total estimated cost (this run): ${total_cost:.4f}")
    else:
        print("Total estimated cost: $unknown (model not in pricing table)")

    return benchmark


def _resolve_model(
    args: argparse.Namespace, config: EvalSuiteConfig, provider: Provider
) -> str:
    """Resolve the model name: ``--model`` > ``EVAL_MODEL`` > preset > provider default."""
    if args.model:
        return str(args.model)
    env_model = os.getenv("EVAL_MODEL")
    if env_model:
        return env_model
    if args.command == "iterate":
        return config.models.small or provider.small_model
    if args.command == "validate":
        return config.models.large or provider.large_model
    return config.models.default or provider.default_model


def _resolve_grader_model(
    args: argparse.Namespace, config: EvalSuiteConfig, agent_model_name: str
) -> str:
    """Resolve the grader model: ``--grader-model`` > ``EVAL_GRADER_MODEL`` > ``[models].grader`` > agent model."""
    if args.grader_model:
        return str(args.grader_model)
    env_grader = os.getenv("EVAL_GRADER_MODEL")
    if env_grader:
        return env_grader
    return config.models.grader or agent_model_name


def _build_parser() -> argparse.ArgumentParser:
    """Construct the argument parser with one subcommand per workflow preset."""
    parser = argparse.ArgumentParser(prog="skill-eval", description="Skill eval runner")
    subparsers = parser.add_subparsers(dest="command", required=True)

    for name in COMMANDS:
        sub = subparsers.add_parser(name, help=_COMMAND_HELP[name])
        sub.add_argument("--case", help="Run only this eval case by id")
        sub.add_argument(
            "--model", help="Agent model name (overrides EVAL_MODEL and presets)"
        )
        sub.add_argument(
            "--grader-model",
            dest="grader_model",
            help="Grader model name (overrides EVAL_GRADER_MODEL and [models].grader; "
            "defaults to the agent model)",
        )
        sub.add_argument(
            "--provider",
            default="anthropic",
            choices=provider_names(),
            help="Model provider (default: anthropic)",
        )
        sub.add_argument(
            "--config",
            type=Path,
            help="Path to skill-eval.toml or its directory (default: discover from cwd)",
        )
        sub.add_argument(
            "--iter",
            type=int,
            dest="iter_force",
            help="Force iteration number (default: auto-increment)",
        )
        sub.add_argument(
            "-d", "--debug", action="store_true", help="Enable debug logging"
        )
        if name == "run":
            sub.add_argument(
                "--no-baseline", action="store_true", help="Skip without_skill runs"
            )
            sub.add_argument(
                "--grade-only",
                action="store_true",
                help="Re-grade existing outputs, skip agent runs",
            )

    return parser


_COMMAND_HELP = {
    "iterate": "Tune cheaply: small model, skill-only (no baseline)",
    "validate": "Sign off: large model, with the without-skill baseline",
    "run": "Raw run with explicit --no-baseline / --grade-only flags",
    "grade": "Re-grade an existing iteration's outputs (requires --iter)",
}


def main(argv: list[str] | None = None) -> None:
    """Entry point for the ``skill-eval`` console script."""
    args = _build_parser().parse_args(argv)

    logging.basicConfig(
        level=logging.DEBUG if args.debug else logging.WARNING,
        format="%(levelname)s %(name)s: %(message)s",
        stream=sys.stderr,
    )

    if args.command == "grade" and args.iter_force is None:
        raise SystemExit("grade requires --iter N (the iteration to re-grade)")

    config = load_config(args.config)
    provider = get_provider(args.provider)
    model_name = _resolve_model(args, config, provider)
    grader_model_name = _resolve_grader_model(args, config, model_name)

    no_baseline = args.command == "iterate" or (
        args.command == "run" and args.no_baseline
    )
    grade_only_mode = args.command == "grade" or (
        args.command == "run" and args.grade_only
    )

    run_suite(
        config,
        provider,
        model_name,
        grader_model_name,
        case_id=args.case,
        no_baseline=no_baseline,
        grade_only_mode=grade_only_mode,
        iter_force=args.iter_force,
    )


if __name__ == "__main__":
    main()