#!/usr/bin/env python3
"""
Eval runner using the GitHub Models API.

GitHub Models provides an OpenAI-compatible endpoint backed by your GitHub
Copilot subscription. It requires a fine-grained Personal Access Token (PAT)
with the `models:read` permission — NOT a classic PAT or OAuth token.

How to create the token:
  1. github.com → Settings → Developer settings → Fine-grained tokens
  2. Generate new token → Permissions → Models → Read-only
  3. export GITHUB_TOKEN=github_pat_...

Usage:
    python copilot_runner.py [options]

    # Set GITHUB_MODEL_API_TOKEN in runner/.env, then:
    python copilot_runner.py
    EVAL_MODEL=openai/gpt-4.1-mini              python copilot_runner.py --no-baseline

Available models (via https://models.github.ai/catalog/models):
    openai/gpt-4.1 (default)
    openai/gpt-4.1-mini
    openai/gpt-4o
    openai/gpt-4o-mini
    meta/llama-4-maverick-17b-128e-instruct-fp8
    meta/llama-3.3-70b-instruct
    See: https://github.com/marketplace/models for the full catalog
"""

import sys
from pathlib import Path

from environs import Env
from langchain_openai import ChatOpenAI

# runner_common.py lives in the same directory
sys.path.insert(0, str(__import__("pathlib").Path(__file__).parent))
from runner_common import main

_env = Env()
_env.read_env(Path(__file__).parent / ".env")

GITHUB_MODELS_BASE_URL = "https://models.github.ai/inference"
DEFAULT_MODEL = "openai/gpt-4.1"


def make_model(model_name: str) -> ChatOpenAI:
    token = _env.str("GITHUB_MODEL_API_TOKEN", None)
    if not token:
        raise SystemExit(
            "GITHUB_MODEL_API_TOKEN is not set.\n\n"
            "Create a fine-grained PAT with Models → Read-only permission:\n"
            "  github.com → Settings → Developer settings → Fine-grained tokens\n\n"
            "Then add it to runner/.env or: export GITHUB_MODEL_API_TOKEN=github_pat_..."
        )
    return ChatOpenAI(
        model=model_name,
        base_url=GITHUB_MODELS_BASE_URL,
        api_key=token,
        max_tokens=8192,
    )


if __name__ == "__main__":
    main(make_model, default_model=DEFAULT_MODEL)