"""Unit tests for the eval service."""

import pytest
from pytest_mock import MockerFixture

from ai_evals.api.schemas.eval import (
    EvalRequest,
    JudgeConfig,
    JudgeResult,
    PipelineContext,
    PromptConfig,
    PromptRunResult,
)
from ai_evals.clients.llm import LLMResult
from ai_evals.services import eval as eval_service


def test_run_prompt_opens_one_mcp_session_and_wires_tool_executor(
    mocker: MockerFixture,
) -> None:
    session = mocker.MagicMock()
    session.__enter__.return_value = session
    session.list_tools.return_value = [
        {"name": "search", "description": "", "inputSchema": {}}
    ]
    session_cls = mocker.patch(
        "ai_evals.services.eval.mcp.McpSession", return_value=session
    )
    run_prompt = mocker.patch(
        "ai_evals.services.eval.llm.run_prompt",
        return_value=LLMResult(tools_called=["search"], answer_text="the answer"),
    )

    result = eval_service._run_prompt(
        prompt="find x", mcp_endpoint="http://localhost:8080/mcp", token="test-jwt"
    )

    session_cls.assert_called_once_with("http://localhost:8080/mcp", "test-jwt")
    run_prompt.assert_called_once_with(
        prompt="find x",
        tools=[{"name": "search", "description": "", "inputSchema": {}}],
        tool_executor=session.call_tool,
    )
    assert result.tools_called == ["search"]


def _pipeline() -> PipelineContext:
    return PipelineContext(pipeline_name="test-pipeline", pipeline_id="123")


def _request(prompts: dict[str, PromptConfig], runs: int = 3) -> EvalRequest:
    return EvalRequest(
        mcp_name="test-mcp",
        mcp_endpoint="http://localhost:8080/mcp",
        runs=runs,
        auth_user="standard_artist",
        pipeline=_pipeline(),
        prompts=prompts,
    )


def test_run_eval_tool_pass_rate_is_one_when_expected_tool_matches(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_pass_rate_is_zero_when_expected_tool_does_not_match(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["other_tool"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 0.0


def test_run_eval_tool_correct_when_no_tool_expected_and_none_called(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=[], answer_text="hello!"),
    )
    request = _request({"p1": PromptConfig(prompt="say hello", expected_tools=[])})

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_correct_when_all_tools_in_multi_step_sequence_are_expected(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(
            tools_called=["search", "execute"], answer_text="14,661 views"
        ),
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="get campaign views", expected_tools=["search", "execute"]
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_correct_when_expected_tool_called_amid_exploration_noise(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(
            tools_called=["introspect", "introspect", "search", "validate", "execute"],
            answer_text="14,661 views",
        ),
    )
    request = _request(
        {"p1": PromptConfig(prompt="get campaign views", expected_tools=["execute"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_incorrect_when_an_expected_tool_was_never_called(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(
            tools_called=["introspect", "introspect", "search", "validate", "execute"],
            answer_text="14,661 views",
        ),
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="get campaign views",
                expected_tools=["execute", "search", "badtool", "validate"],
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 0.0


def test_run_eval_tool_incorrect_when_none_of_the_called_tools_are_expected(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["introspect", "validate"], answer_text=""),
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="get campaign views", expected_tools=["search", "execute"]
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 0.0


def test_run_eval_no_judges_configured_gives_empty_pass_rates(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].judge_pass_rates == {}


def test_run_eval_judge_pass_rate_reflects_judge_result(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text="the answer"),
    )
    mocker.patch(
        "ai_evals.services.eval._run_judge",
        return_value=JudgeResult(name="correct_artist", passed=True),
    )
    judge = JudgeConfig(
        name="correct_artist", instructions="does it say the right artist?"
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="find x", expected_tools=["stub_tool"], judges=[judge]
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].judge_pass_rates == {"correct_artist": 1.0}


def test_run_eval_judge_pass_rate_reflects_mixed_results(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text="the answer"),
    )
    judge = JudgeConfig(
        name="correct_artist", instructions="does it say the right artist?"
    )
    mocker.patch(
        "ai_evals.services.eval._run_judge",
        side_effect=[
            JudgeResult(name="correct_artist", passed=True),
            JudgeResult(name="correct_artist", passed=False),
        ],
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="find x", expected_tools=["stub_tool"], judges=[judge]
            )
        },
        runs=2,
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].judge_pass_rates == {"correct_artist": 0.5}


def test_run_eval_uses_top_level_runs_by_default(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"])}, runs=4
    )

    response = eval_service.run_eval(request)

    assert len(response.prompts["p1"].runs) == 4


def test_run_eval_prompt_level_runs_overrides_top_level(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_evals.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"], runs=1)},
        runs=4,
    )

    response = eval_service.run_eval(request)

    assert len(response.prompts["p1"].runs) == 1


def test_run_eval_multiple_prompts_produce_independent_results(
    mocker: MockerFixture,
) -> None:
    def _side_effect(prompt: str, mcp_endpoint: str, token: str) -> LLMResult:
        if prompt == "find x":
            return LLMResult(tools_called=["stub_tool"], answer_text="")
        return LLMResult(tools_called=["unmatched_tool"], answer_text="")

    mocker.patch("ai_evals.services.eval._run_prompt", side_effect=_side_effect)
    request = _request(
        {
            "p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"]),
            "p2": PromptConfig(prompt="find y", expected_tools=["other_tool"]),
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0
    assert response.prompts["p2"].tool_pass_rate == 0.0


def test_aggregate_prompt_results_computes_tool_pass_rate() -> None:
    run_results = [
        PromptRunResult(tools_called=["a"], tool_correct=True, judge_results=[]),
        PromptRunResult(tools_called=["b"], tool_correct=False, judge_results=[]),
    ]

    result = eval_service._aggregate_prompt_results(run_results, runs=2)

    assert result.tool_pass_rate == 0.5
    assert result.judge_pass_rates == {}


def test_aggregate_prompt_results_computes_judge_pass_rate_per_name() -> None:
    run_results = [
        PromptRunResult(
            tools_called=["a"],
            tool_correct=True,
            judge_results=[
                JudgeResult(name="j1", passed=True),
                JudgeResult(name="j2", passed=False),
            ],
        ),
        PromptRunResult(
            tools_called=["a"],
            tool_correct=True,
            judge_results=[
                JudgeResult(name="j1", passed=True),
                JudgeResult(name="j2", passed=True),
            ],
        ),
    ]

    result = eval_service._aggregate_prompt_results(run_results, runs=2)

    assert result.judge_pass_rates == {"j1": 1.0, "j2": 0.5}


def test_resolve_auth_token_prefers_request_level_token() -> None:
    request = EvalRequest(
        mcp_name="test-mcp",
        mcp_endpoint="http://localhost:8080/mcp",
        auth_token="request-jwt",
        pipeline=_pipeline(),
        prompts={},
    )
    prompt_config = PromptConfig(prompt="find x", expected_tools=[])

    token = eval_service._resolve_auth_token(prompt_config, request)

    assert token == "request-jwt"


def test_resolve_auth_token_prompt_level_overrides_request_level() -> None:
    request = EvalRequest(
        mcp_name="test-mcp",
        mcp_endpoint="http://localhost:8080/mcp",
        auth_token="request-jwt",
        pipeline=_pipeline(),
        prompts={},
    )
    prompt_config = PromptConfig(
        prompt="find x", expected_tools=[], auth_token="prompt-jwt"
    )

    token = eval_service._resolve_auth_token(prompt_config, request)

    assert token == "prompt-jwt"


def test_resolve_auth_token_falls_back_to_auth_user_ropg_stub(
    mocker: MockerFixture,
) -> None:
    mocker.patch("ai_evals.services.eval._get_auth_token", return_value="stub-token")
    request = EvalRequest(
        mcp_name="test-mcp",
        mcp_endpoint="http://localhost:8080/mcp",
        auth_user="standard_artist",
        pipeline=_pipeline(),
        prompts={},
    )
    prompt_config = PromptConfig(prompt="find x", expected_tools=[])

    token = eval_service._resolve_auth_token(prompt_config, request)

    assert token == "stub-token"


def test_resolve_auth_token_raises_when_neither_token_nor_user_set() -> None:
    request = EvalRequest(
        mcp_name="test-mcp",
        mcp_endpoint="http://localhost:8080/mcp",
        pipeline=_pipeline(),
        prompts={},
    )
    prompt_config = PromptConfig(prompt="find x", expected_tools=[])

    with pytest.raises(ValueError, match="auth_token or auth_user"):
        eval_service._resolve_auth_token(prompt_config, request)


def test_tool_selection_reasoning_when_expected_tool_matched_amid_noise() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=["introspect", "search", "execute"],
        expected_tools=["execute"],
        tool_correct=True,
    )

    assert "execute" in reasoning
    assert "introspect, search, execute" in reasoning


def test_tool_selection_reasoning_when_no_expected_tool_matched() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=["introspect", "validate"],
        expected_tools=["search", "execute"],
        tool_correct=False,
    )

    assert "introspect, validate" in reasoning
    assert "search, execute" in reasoning


def test_tool_selection_reasoning_when_no_tools_expected_and_none_called() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=[], expected_tools=[], tool_correct=True
    )

    assert "no tool calls" in reasoning.lower()


def test_tool_selection_reasoning_when_no_tools_expected_but_some_called() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=["search"], expected_tools=[], tool_correct=False
    )

    assert "search" in reasoning