"""Job orchestration — runs eval jobs in the background and tracks their status.""" import asyncio import uuid from ai_evals.api.schemas.eval import EvalRequest, EvalResponse, JobResult, JobStatus from ai_evals.clients import jobs as jobs_client from ai_evals.services.eval import run_eval def _execute(job_id: str, request: EvalRequest) -> None: jobs_client.update_job(job_id, status=JobStatus.RUNNING) try: result = run_eval(request) except Exception as exc: # noqa: BLE001 — recorded on the job, not raised to a caller jobs_client.update_job(job_id, status=JobStatus.FAILED, error=str(exc)) return jobs_client.update_job( job_id, status=JobStatus.SUCCEEDED, result_json=result.model_dump_json() ) # Keeps a strong reference to in-flight background tasks so they aren't garbage collected # before completion — asyncio only holds a weak reference once nothing else does. _background_tasks: set[asyncio.Task] = set() async def start_job(request: EvalRequest) -> str: job_id = str(uuid.uuid4()) jobs_client.create_job( job_id, mcp_name=request.mcp_name, pipeline_id=request.pipeline.pipeline_id ) task = asyncio.create_task(asyncio.to_thread(_execute, job_id, request)) _background_tasks.add(task) task.add_done_callback(_background_tasks.discard) return job_id def get_job(job_id: str) -> JobResult | None: record = jobs_client.get_job(job_id) if record is None: return None result = ( EvalResponse.model_validate_json(record.result_json) if record.result_json else None ) return JobResult( job_id=job_id, status=JobStatus(record.status), result=result, error=record.error, )