import math import uuid from typing import Annotated, Any from fastapi import APIRouter, HTTPException, Query from app.adapters.db import db from app.api.pipeline import schemas from app.config import settings from app.dsp.gateway import dsp_gateway from app.dsp.models import DSPClient from app.fandata.models import ( Fan, FanCredentials, FanCredentialsFilter, FanRecentlyPlayed, FanStepState, FanTopArtist, ) from app.pipeline import dispatch, services from app.pipeline.enums import RunSource, RunStatus from app.pipeline.exceptions import PipelineAlreadyRunningError from app.pipeline.models import PipelineRun router = APIRouter(prefix="/pipeline", tags=["Pipeline"]) REQUESTS_PER_FAN = len(FanStepState.model_fields) + 1 # steps + token refresh @router.get("/stats", response_model=schemas.DashboardStats) def get_dashboard_stats() -> Any: with db.autocommit(): collected_today = Fan.query.collected_today() top_artists_total = FanTopArtist.query.count() recently_played_total = FanRecentlyPlayed.query.count() active_run = ( PipelineRun.query.where( PipelineRun.status.in_( [RunStatus.running, RunStatus.paused, RunStatus.queued] ) ) .order_by(PipelineRun.started_at.desc()) .first() ) last_run = ( PipelineRun.query.where(PipelineRun.finished_at.isnot(None)) .order_by(PipelineRun.finished_at.desc()) .first() ) throughput: int | None = None throughput_rps: float | None = None rate_limited_last_run: int | None = None stale_tokens_last_run: int | None = None if last_run: rate_limited_last_run = last_run.rate_limited stale_tokens_last_run = last_run.stale_tokens fps = last_run.throughput_fps throughput = round(fps) if fps is not None else None throughput_rps = last_run.throughput_rps return { "collected_today": collected_today, "top_artists_total": top_artists_total, "recently_played_total": recently_played_total, "throughput": throughput, "throughput_rps": throughput_rps, "rate_limited_last_run": rate_limited_last_run, "stale_tokens_last_run": stale_tokens_last_run, "active_run_id": str(active_run.id) if active_run else None, } @router.post( "/run", response_model=schemas.PipelineRun, status_code=201, ) def start_pipeline_run(data: schemas.StartPipelineInput) -> Any: try: with db.transaction(): run = services.create_queued_run( source=RunSource.manual, dsp_client_id=data.dsp_client_id, token_status=data.token_status, not_collected_since=data.not_collected_since, max_consecutive_failures=data.max_consecutive_failures, limit=data.limit, ) except PipelineAlreadyRunningError as exc: raise HTTPException(status_code=409, detail=str(exc)) from exc dispatch.dispatch_fan_fanout(run.id) return run @router.get( "/runs", response_model=schemas.PipelineRunPaginated, ) def list_pipeline_runs( limit: Annotated[int, Query(ge=1, le=100)] = 20, offset: Annotated[int, Query(ge=0)] = 0, statuses: Annotated[list[RunStatus] | None, Query()] = None, ) -> Any: with db.autocommit(): q = PipelineRun.query if statuses: q = q.where(PipelineRun.status.in_(statuses)) result = q.order_by(PipelineRun.created_at.desc()).paginate( limit=limit, offset=offset ) return schemas.PipelineRunPaginated( items=[ schemas.PipelineRun.model_validate(r, from_attributes=True) for r in result.items ], total=result.total, ) @router.get( "/estimate", response_model=schemas.EstimateResponse, ) def estimate_pipeline_run( params: Annotated[schemas.StartPipelineInput, Query()], ) -> Any: with db.autocommit(): filters = FanCredentialsFilter(dsp_client_id=params.dsp_client_id) total_fans = FanCredentials.query.filter(filters).count() if params.limit: total_fans = min(total_fans, params.limit) client = DSPClient.query.where( DSPClient.id == params.dsp_client_id ).one_or_none() effective_rps: float | None = None if client is not None: live_stats = dsp_gateway.stats(client.name) if live_stats is not None and live_stats.rps is not None: effective_rps = live_stats.rps elif client.nominal_rps is not None: max_rps_by_concurrency = ( settings.fan_collect_concurrency_for(client.name) * 1000 / settings.pipeline_run_estimate_avg_latency_ms ) effective_rps = min(client.nominal_rps, max_rps_by_concurrency) estimated_seconds: int | None = None if total_fans > 0 and effective_rps and effective_rps > 0: estimated_seconds = math.ceil(total_fans * REQUESTS_PER_FAN / effective_rps) return { "total_fans": total_fans, "estimated_seconds": estimated_seconds, "requests_per_fan": REQUESTS_PER_FAN, "throughput_rps": round(effective_rps) if effective_rps is not None else None, } @router.get( "/runs/{run_id}", response_model=schemas.PipelineRun, ) def get_pipeline_run(run_id: uuid.UUID) -> Any: with db.autocommit(): run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return schemas.PipelineRun.model_validate(run, from_attributes=True) @router.post( "/runs/{run_id}/pause", response_model=schemas.PipelineRun, ) def pause_pipeline_run(run_id: uuid.UUID) -> Any: with db.transaction(): run = services.pause_run(run_id) if run is None: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return run @router.post( "/runs/{run_id}/resume", response_model=schemas.PipelineRun, ) def resume_pipeline_run(run_id: uuid.UUID) -> Any: with db.transaction(): run = services.resume_run(run_id) if run is None: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return run @router.post( "/runs/{run_id}/cancel", response_model=schemas.PipelineRun, ) def cancel_pipeline_run(run_id: uuid.UUID) -> Any: with db.transaction(): run = services.cancel_run(run_id) if run is None: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return run @router.post( "/runs/{run_id}/stall", response_model=schemas.PipelineRun, ) def stall_pipeline_run(run_id: uuid.UUID) -> Any: with db.transaction(): run = services.stall_run(run_id) if run is None: raise HTTPException(status_code=404, detail=f"Run {run_id} not found") return run @router.post( "/runs/cleanup", response_model=schemas.CleanupStaleRunsResponse, ) def cleanup_stale_runs( dsp_client_id: Annotated[int | None, Query()] = None, ) -> Any: with db.transaction(): if dsp_client_id is not None: cleaned = services.cleanup_stale_runs(dsp_client_id) else: cleaned = [] for client in DSPClient.query.all(): cleaned.extend(services.cleanup_stale_runs(client.id)) return schemas.CleanupStaleRunsResponse( cleaned=[str(r) for r in cleaned], count=len(cleaned), )