"""Eval schemas."""

from enum import StrEnum

from pydantic import BaseModel, Field

# Datadog LLM Obs evaluation labels must start with a letter and contain only
# ASCII alphanumerics/underscores, max 200 chars. judge.name is submitted
# directly as the evaluation label, so it must satisfy this constraint.
_DD_EVAL_LABEL_PATTERN = r"^[A-Za-z][A-Za-z0-9_]{0,199}$"


class JudgeConfig(BaseModel):
    name: str = Field(
        pattern=_DD_EVAL_LABEL_PATTERN,
        description=(
            "Submitted directly as the Datadog LLM Obs evaluation label. "
            "Must start with a letter and contain only letters, numbers, "
            "and underscores (max 200 chars)."
        ),
    )
    instructions: str


class PromptConfig(BaseModel):
    prompt: str
    expected_tools: list[str]
    judges: list[JudgeConfig] | None = None
    runs: int | None = None
    auth_user: str | None = None
    auth_token: str | None = None


class PipelineContext(BaseModel):
    pipeline_name: str
    pipeline_id: str
    build_id: str | None = None


class EvalRequest(BaseModel):
    mcp_name: str
    mcp_endpoint: str
    runs: int = 3
    auth_user: str | None = None
    auth_token: str | None = None
    pipeline: PipelineContext
    prompts: dict[str, PromptConfig]


class JudgeResult(BaseModel):
    name: str
    passed: bool
    rationale: str | None = None


class PromptRunResult(BaseModel):
    tools_called: list[str]
    tool_correct: bool
    judge_results: list[JudgeResult]


class PromptResult(BaseModel):
    tool_pass_rate: float
    judge_pass_rates: dict[str, float]
    runs: list[PromptRunResult]


class EvalResponse(BaseModel):
    mcp: str
    model: str
    prompts: dict[str, PromptResult]


class JobStatus(StrEnum):
    PENDING = "pending"
    RUNNING = "running"
    SUCCEEDED = "succeeded"
    FAILED = "failed"


class JobAccepted(BaseModel):
    job_id: str
    status: JobStatus


class JobResult(BaseModel):
    job_id: str
    status: JobStatus
    result: EvalResponse | None = None
    error: str | None = None