"""Unit tests for the Bedrock LLM client's tool-calling loop.""" from unittest.mock import MagicMock from pytest_mock import MockerFixture from ai_eval_runner.clients import llm _TOOLS = [{"name": "search", "description": "", "inputSchema": {"type": "object"}}] def _text_response(text: str) -> dict: return {"output": {"message": {"role": "assistant", "content": [{"text": text}]}}} def _tool_use_response( tool_use_id: str, name: str, tool_input: dict | None = None ) -> dict: return { "output": { "message": { "role": "assistant", "content": [ { "toolUse": { "toolUseId": tool_use_id, "name": name, "input": tool_input or {}, } } ], } } } def _mock_bedrock(mocker: MockerFixture, responses: list[dict]) -> MagicMock: bedrock = mocker.MagicMock() bedrock.converse.side_effect = responses mocker.patch("ai_eval_runner.clients.llm._bedrock_client", return_value=bedrock) return bedrock def test_run_prompt_returns_immediately_when_no_tool_is_called( mocker: MockerFixture, ) -> None: _mock_bedrock(mocker, [_text_response("hello!")]) result = llm.run_prompt(prompt="say hello", tools=[]) assert result == llm.LLMResult(tools_called=[], answer_text="hello!") def test_run_prompt_does_not_execute_tool_when_no_tool_executor_given( mocker: MockerFixture, ) -> None: _mock_bedrock(mocker, [_tool_use_response("t1", "search")]) result = llm.run_prompt(prompt="find x", tools=_TOOLS) assert result.tools_called == [] def test_run_prompt_iterates_across_multiple_tool_calls_until_final_answer( mocker: MockerFixture, ) -> None: bedrock = _mock_bedrock( mocker, [ _tool_use_response("t1", "search", {"query": "campaign views"}), _tool_use_response("t2", "execute", {"query": "..."}), _text_response("14,661 views"), ], ) executor = mocker.MagicMock(side_effect=["found campaign", "14661"]) result = llm.run_prompt( prompt="get campaign views", tools=_TOOLS, tool_executor=executor ) assert result.tools_called == ["search", "execute"] assert result.answer_text == "14,661 views" assert executor.call_args_list == [ mocker.call("search", {"query": "campaign views"}), mocker.call("execute", {"query": "..."}), ] assert bedrock.converse.call_count == 3 def test_run_prompt_feeds_tool_executor_error_back_as_a_failed_tool_result( mocker: MockerFixture, ) -> None: _mock_bedrock( mocker, [ _tool_use_response("t1", "execute", {"query": "bad"}), _text_response("sorry, that failed"), ], ) executor = mocker.MagicMock(side_effect=RuntimeError("mcp unreachable")) result = llm.run_prompt(prompt="do it", tools=_TOOLS, tool_executor=executor) assert result.tools_called == ["execute"] assert result.answer_text == "sorry, that failed" def test_run_prompt_stops_after_max_turns_even_without_a_final_answer( mocker: MockerFixture, ) -> None: bedrock = _mock_bedrock( mocker, [ _tool_use_response("t1", "search"), _tool_use_response("t2", "search"), ], ) executor = mocker.MagicMock(return_value="ok") result = llm.run_prompt( prompt="loop forever", tools=_TOOLS, tool_executor=executor, max_turns=2 ) assert result.tools_called == ["search", "search"] assert bedrock.converse.call_count == 2 assert result.max_turns_exceeded is True def test_run_prompt_does_not_flag_max_turns_exceeded_when_it_finishes_on_the_last_turn( mocker: MockerFixture, ) -> None: _mock_bedrock( mocker, [ _tool_use_response("t1", "search"), _text_response("done"), ], ) executor = mocker.MagicMock(return_value="ok") result = llm.run_prompt( prompt="just in time", tools=_TOOLS, tool_executor=executor, max_turns=2 ) assert result.answer_text == "done" assert result.max_turns_exceeded is False def test_judge_tool_input_schema_stays_in_sync_with_judge_verdict_fields() -> None: # _JUDGE_TOOL_INPUT_SCHEMA is a hand-written Bedrock tool schema describing the same # shape as JudgeVerdict — this pins them together so a rename/field change fails here # instead of surfacing as a runtime KeyError in run_judge or a Bedrock schema rejection. assert set(llm._JUDGE_TOOL_INPUT_SCHEMA["properties"]) == set( llm.JudgeVerdict._fields ) assert set(llm._JUDGE_TOOL_INPUT_SCHEMA["required"]) == set( llm.JudgeVerdict._fields )