"""eval router unit tests.""" from fastapi.testclient import TestClient from pytest_mock import MockerFixture from ai_evals.api.schemas.eval import EvalResponse, JobResult, JobStatus _VALID_PAYLOAD = { "mcp_name": "test-mcp", "mcp_endpoint": "http://localhost:8080/mcp", "auth_token": "test-jwt", "pipeline": {"pipeline_name": "test-pipeline", "pipeline_id": "123"}, "prompts": { "p1": { "prompt": "find the top artist", "expected_tools": ["stub_tool"], } }, } def test_run_eval_requires_api_key(test_client: TestClient) -> None: response = test_client.post("/run-eval", json=_VALID_PAYLOAD) assert response.status_code == 401 def test_run_eval_rejects_invalid_api_key(test_client: TestClient) -> None: response = test_client.post( "/run-eval", json=_VALID_PAYLOAD, headers={"X-API-Key": "wrong-key"} ) assert response.status_code == 403 def test_run_eval_accepts_request_and_returns_job_id( test_client: TestClient, mocker: MockerFixture ) -> None: mocker.patch("ai_evals.api.routers.eval.jobs.start_job", return_value="job-1") response = test_client.post( "/run-eval", json=_VALID_PAYLOAD, headers={"X-API-Key": "dev-key"} ) assert response.status_code == 202 assert response.json() == {"job_id": "job-1", "status": "pending"} def test_run_eval_rejects_missing_required_field(test_client: TestClient) -> None: payload = {k: v for k, v in _VALID_PAYLOAD.items() if k != "pipeline"} response = test_client.post( "/run-eval", json=payload, headers={"X-API-Key": "dev-key"} ) assert response.status_code == 422 def test_run_eval_rejects_judge_name_starting_with_number( test_client: TestClient, ) -> None: payload = { **_VALID_PAYLOAD, "prompts": { "p1": { "prompt": "find the top artist", "expected_tools": ["stub_tool"], "judges": [{"name": "1_bad_name", "instructions": "check something"}], } }, } response = test_client.post( "/run-eval", json=payload, headers={"X-API-Key": "dev-key"} ) assert response.status_code == 422 def test_run_eval_rejects_judge_name_with_invalid_characters( test_client: TestClient, ) -> None: payload = { **_VALID_PAYLOAD, "prompts": { "p1": { "prompt": "find the top artist", "expected_tools": ["stub_tool"], "judges": [{"name": "bad-name!", "instructions": "check something"}], } }, } response = test_client.post( "/run-eval", json=payload, headers={"X-API-Key": "dev-key"} ) assert response.status_code == 422 def test_run_eval_accepts_valid_judge_name( test_client: TestClient, mocker: MockerFixture ) -> None: mocker.patch("ai_evals.api.routers.eval.jobs.start_job", return_value="job-1") payload = { **_VALID_PAYLOAD, "prompts": { "p1": { "prompt": "find the top artist", "expected_tools": ["stub_tool"], "judges": [ {"name": "correct_artist_2", "instructions": "check something"} ], } }, } response = test_client.post( "/run-eval", json=payload, headers={"X-API-Key": "dev-key"} ) assert response.status_code == 202 def test_get_eval_job_requires_api_key(test_client: TestClient) -> None: response = test_client.get("/run-eval/job-1") assert response.status_code == 401 def test_get_eval_job_returns_404_when_missing( test_client: TestClient, mocker: MockerFixture ) -> None: mocker.patch("ai_evals.api.routers.eval.jobs.get_job", return_value=None) response = test_client.get("/run-eval/missing", headers={"X-API-Key": "dev-key"}) assert response.status_code == 404 def test_get_eval_job_returns_result_when_succeeded( test_client: TestClient, mocker: MockerFixture ) -> None: result = JobResult( job_id="job-1", status=JobStatus.SUCCEEDED, result=EvalResponse(mcp="test-mcp", model="test-model", prompts={}), error=None, ) mocker.patch("ai_evals.api.routers.eval.jobs.get_job", return_value=result) response = test_client.get("/run-eval/job-1", headers={"X-API-Key": "dev-key"}) assert response.status_code == 200 body = response.json() assert body["job_id"] == "job-1" assert body["status"] == "succeeded" assert body["result"]["mcp"] == "test-mcp"