"""Unit tests for the eval service."""

from pytest_mock import MockerFixture

from ai_eval_runner.api.schemas.eval import (
    EvalRequest,
    JudgeConfig,
    JudgeResult,
    McpConfig,
    PipelineContext,
    PromptConfig,
    PromptRunResult,
)
from ai_eval_runner.clients.llm import LLMResult
from ai_eval_runner.services import eval as eval_service


def test_run_prompt_opens_one_mcp_session_and_wires_tool_executor(
    mocker: MockerFixture,
) -> None:
    session = mocker.MagicMock()
    session.__enter__.return_value = session
    session.list_tools.return_value = [
        {"name": "search", "description": "", "inputSchema": {}}
    ]
    session_cls = mocker.patch(
        "ai_eval_runner.services.eval.mcp.McpSession", return_value=session
    )
    run_prompt = mocker.patch(
        "ai_eval_runner.services.eval.llm.run_prompt",
        return_value=LLMResult(tools_called=["search"], answer_text="the answer"),
    )

    result = eval_service._run_prompt(
        prompt="find x",
        mcp_endpoint="http://localhost:8080/mcp",
        token="test-jwt",
        max_turns=None,
    )

    session_cls.assert_called_once_with("http://localhost:8080/mcp", "test-jwt")
    run_prompt.assert_called_once_with(
        prompt="find x",
        tools=[{"name": "search", "description": "", "inputSchema": {}}],
        tool_executor=session.call_tool,
        max_turns=None,
    )
    assert result.tools_called == ["search"]


def test_run_prompt_passes_max_turns_through_to_llm_run_prompt(
    mocker: MockerFixture,
) -> None:
    session = mocker.MagicMock()
    session.__enter__.return_value = session
    session.list_tools.return_value = []
    mocker.patch("ai_eval_runner.services.eval.mcp.McpSession", return_value=session)
    run_prompt = mocker.patch(
        "ai_eval_runner.services.eval.llm.run_prompt",
        return_value=LLMResult(tools_called=[], answer_text=""),
    )

    eval_service._run_prompt(
        prompt="find x",
        mcp_endpoint="http://localhost:8080/mcp",
        token="test-jwt",
        max_turns=3,
    )

    run_prompt.assert_called_once_with(
        prompt="find x", tools=[], tool_executor=session.call_tool, max_turns=3
    )


def _pipeline() -> PipelineContext:
    return PipelineContext(pipeline_name="test-pipeline", pipeline_id="123")


def _request(prompts: dict[str, PromptConfig], runs: int = 3) -> EvalRequest:
    return EvalRequest(
        mcp=McpConfig(
            name="test-mcp",
            endpoint="http://localhost:8080/mcp",
            auth_token="test-jwt",
        ),
        runs=runs,
        pipeline=_pipeline(),
        prompts=prompts,
    )


def test_run_eval_tool_pass_rate_is_one_when_expected_tool_matches(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_pass_rate_is_zero_when_expected_tool_does_not_match(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["other_tool"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 0.0


def test_run_eval_tool_correct_when_no_tool_expected_and_none_called(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=[], answer_text="hello!"),
    )
    request = _request({"p1": PromptConfig(prompt="say hello", expected_tools=[])})

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_correct_when_all_tools_in_multi_step_sequence_are_expected(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(
            tools_called=["search", "execute"], answer_text="14,661 views"
        ),
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="get campaign views", expected_tools=["search", "execute"]
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_correct_when_expected_tool_called_amid_exploration_noise(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(
            tools_called=["introspect", "introspect", "search", "validate", "execute"],
            answer_text="14,661 views",
        ),
    )
    request = _request(
        {"p1": PromptConfig(prompt="get campaign views", expected_tools=["execute"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_run_eval_tool_incorrect_when_an_expected_tool_was_never_called(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(
            tools_called=["introspect", "introspect", "search", "validate", "execute"],
            answer_text="14,661 views",
        ),
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="get campaign views",
                expected_tools=["execute", "search", "badtool", "validate"],
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 0.0


def test_run_eval_tool_incorrect_when_none_of_the_called_tools_are_expected(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["introspect", "validate"], answer_text=""),
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="get campaign views", expected_tools=["search", "execute"]
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 0.0


def test_run_eval_no_judges_configured_gives_empty_pass_rates(
    mocker: MockerFixture,
) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"])}
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].judge_pass_rates == {}


def test_run_eval_judge_pass_rate_reflects_judge_result(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text="the answer"),
    )
    mocker.patch(
        "ai_eval_runner.services.eval._run_judge",
        return_value=JudgeResult(name="correct_artist", passed=True),
    )
    judge = JudgeConfig(
        name="correct_artist", instructions="does it say the right artist?"
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="find x", expected_tools=["stub_tool"], judges=[judge]
            )
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].judge_pass_rates == {"correct_artist": 1.0}


def test_run_eval_judge_pass_rate_reflects_mixed_results(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text="the answer"),
    )
    judge = JudgeConfig(
        name="correct_artist", instructions="does it say the right artist?"
    )
    mocker.patch(
        "ai_eval_runner.services.eval._run_judge",
        side_effect=[
            JudgeResult(name="correct_artist", passed=True),
            JudgeResult(name="correct_artist", passed=False),
        ],
    )
    request = _request(
        {
            "p1": PromptConfig(
                prompt="find x", expected_tools=["stub_tool"], judges=[judge]
            )
        },
        runs=2,
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].judge_pass_rates == {"correct_artist": 0.5}


def test_run_eval_uses_top_level_runs_by_default(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"])}, runs=4
    )

    response = eval_service.run_eval(request)

    assert len(response.prompts["p1"].runs) == 4


def test_run_eval_prompt_level_runs_overrides_top_level(mocker: MockerFixture) -> None:
    mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=["stub_tool"], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"], runs=1)},
        runs=4,
    )

    response = eval_service.run_eval(request)

    assert len(response.prompts["p1"].runs) == 1


def test_run_eval_passes_prompt_level_max_turns_to_run_prompt(
    mocker: MockerFixture,
) -> None:
    run_prompt = mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=[], answer_text=""),
    )
    request = _request(
        {"p1": PromptConfig(prompt="find x", expected_tools=[], max_turns=3, runs=1)}
    )

    eval_service.run_eval(request)

    run_prompt.assert_called_once_with(
        prompt="find x",
        mcp_endpoint="http://localhost:8080/mcp",
        token=mocker.ANY,
        max_turns=3,
    )


def test_run_eval_defaults_max_turns_to_none_when_unset(mocker: MockerFixture) -> None:
    run_prompt = mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=[], answer_text=""),
    )
    request = _request({"p1": PromptConfig(prompt="find x", expected_tools=[], runs=1)})

    eval_service.run_eval(request)

    assert run_prompt.call_args.kwargs["max_turns"] is None


def test_run_eval_multiple_prompts_produce_independent_results(
    mocker: MockerFixture,
) -> None:
    def _side_effect(
        prompt: str, mcp_endpoint: str, token: str, max_turns: int | None
    ) -> LLMResult:
        if prompt == "find x":
            return LLMResult(tools_called=["stub_tool"], answer_text="")
        return LLMResult(tools_called=["unmatched_tool"], answer_text="")

    mocker.patch("ai_eval_runner.services.eval._run_prompt", side_effect=_side_effect)
    request = _request(
        {
            "p1": PromptConfig(prompt="find x", expected_tools=["stub_tool"]),
            "p2": PromptConfig(prompt="find y", expected_tools=["other_tool"]),
        }
    )

    response = eval_service.run_eval(request)

    assert response.prompts["p1"].tool_pass_rate == 1.0
    assert response.prompts["p2"].tool_pass_rate == 0.0


def test_aggregate_prompt_results_computes_tool_pass_rate() -> None:
    run_results = [
        PromptRunResult(
            tools_called=["a"],
            tool_correct=True,
            judge_results=[],
            max_turns_exceeded=False,
        ),
        PromptRunResult(
            tools_called=["b"],
            tool_correct=False,
            judge_results=[],
            max_turns_exceeded=False,
        ),
    ]

    result = eval_service._aggregate_prompt_results(run_results, runs=2)

    assert result.tool_pass_rate == 0.5
    assert result.judge_pass_rates == {}


def test_aggregate_prompt_results_computes_judge_pass_rate_per_name() -> None:
    run_results = [
        PromptRunResult(
            tools_called=["a"],
            tool_correct=True,
            judge_results=[
                JudgeResult(name="j1", passed=True),
                JudgeResult(name="j2", passed=False),
            ],
            max_turns_exceeded=False,
        ),
        PromptRunResult(
            tools_called=["a"],
            tool_correct=True,
            judge_results=[
                JudgeResult(name="j1", passed=True),
                JudgeResult(name="j2", passed=True),
            ],
            max_turns_exceeded=False,
        ),
    ]

    result = eval_service._aggregate_prompt_results(run_results, runs=2)

    assert result.judge_pass_rates == {"j1": 1.0, "j2": 0.5}


def test_run_eval_uses_mcp_auth_token_when_mcp_configured(
    mocker: MockerFixture,
) -> None:
    run_prompt = mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=[], answer_text=""),
    )
    request = _request({"p1": PromptConfig(prompt="find x", runs=1)})

    eval_service.run_eval(request)

    run_prompt.assert_called_once_with(
        prompt="find x",
        mcp_endpoint="http://localhost:8080/mcp",
        token="test-jwt",
        max_turns=None,
    )


def test_run_eval_skips_mcp_entirely_when_mcp_not_set(mocker: MockerFixture) -> None:
    run_prompt = mocker.patch(
        "ai_eval_runner.services.eval._run_prompt",
        return_value=LLMResult(tools_called=[], answer_text="hi"),
    )
    request = EvalRequest(
        pipeline=_pipeline(),
        prompts={"p1": PromptConfig(prompt="say hi", runs=1)},
    )

    response = eval_service.run_eval(request)

    run_prompt.assert_called_once_with(
        prompt="say hi", mcp_endpoint=None, token=None, max_turns=None
    )
    assert response.mcp == eval_service.NO_MCP_NAME
    assert response.prompts["p1"].tool_pass_rate == 1.0


def test_tool_selection_reasoning_when_expected_tool_matched_amid_noise() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=["introspect", "search", "execute"],
        expected_tools=["execute"],
        tool_correct=True,
    )

    assert "execute" in reasoning
    assert "introspect, search, execute" in reasoning


def test_tool_selection_reasoning_when_no_expected_tool_matched() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=["introspect", "validate"],
        expected_tools=["search", "execute"],
        tool_correct=False,
    )

    assert "introspect, validate" in reasoning
    assert "search, execute" in reasoning


def test_tool_selection_reasoning_when_no_tools_expected_and_none_called() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=[], expected_tools=[], tool_correct=True
    )

    assert "no tool calls" in reasoning.lower()


def test_tool_selection_reasoning_when_no_tools_expected_but_some_called() -> None:
    reasoning = eval_service._tool_selection_reasoning(
        tools_called=["search"], expected_tools=[], tool_correct=False
    )

    assert "search" in reasoning