import logging import uuid from collections.abc import Sequence from datetime import datetime, timedelta from fansifter_common.utils import timezone from app.config import settings from app.fandata.enums import FanCredentialsStatus from app.fandata.models import FanCredentialsFilter from app.pipeline.enums import RunFinishedReason, RunSource, RunStatus from app.pipeline.exceptions import ( PipelineAlreadyRunningError, PipelineRunNotFoundError, ) from app.pipeline.models import PipelineRun logger = logging.getLogger(__name__) def _stale_cutoff() -> datetime: return timezone.now() - timedelta(seconds=settings.pipeline_run_stale_timeout_s) def get_stale_runs(dsp_client_id: int | None = None) -> Sequence[PipelineRun]: """Return active runs that have exceeded the stale timeout.""" cutoff = _stale_cutoff() q = PipelineRun.query.active().where(PipelineRun.started_at < cutoff) if dsp_client_id is not None: q = q.where(PipelineRun.dsp_client_id == dsp_client_id) return q.all() def cleanup_stale_runs(dsp_client_id: int) -> list[uuid.UUID]: """Mark active runs that have exceeded the stale timeout as error/stalled.""" stale = get_stale_runs(dsp_client_id) cleaned: list[uuid.UUID] = [] for run in stale: logger.warning( "Marking run %s as stalled (started_at=%s, threshold=%ds)", run.id, run.started_at, settings.pipeline_run_stale_timeout_s, extra={"run_id": str(run.id)}, ) run.status = RunStatus.error run.finished_at = timezone.now() run.finished_reason = RunFinishedReason.stalled run.save() cleaned.append(run.id) return cleaned def _check_no_active_run_for(dsp_client_id: int) -> None: active = ( PipelineRun.query.active() .where(PipelineRun.dsp_client_id == dsp_client_id) .one_or_none() ) if active is not None: raise PipelineAlreadyRunningError(active.id) def create_queued_run( *, source: RunSource, dsp_client_id: int, token_status: FanCredentialsStatus | None = None, not_collected_since: datetime | None = None, max_consecutive_failures: int | None = None, limit: int | None = None, ) -> PipelineRun: cleanup_stale_runs(dsp_client_id) _check_no_active_run_for(dsp_client_id) run = PipelineRun( status=RunStatus.queued, dsp_client_id=dsp_client_id, source=source, filters=FanCredentialsFilter( dsp_client_id=dsp_client_id, token_status=token_status or FanCredentialsStatus.active, not_collected_since=not_collected_since, max_consecutive_failures=max_consecutive_failures, limit=limit or settings.fan_fanout_max_fans, ), ) run.save(flush=True) run.refresh(attribute_names=["dsp_client"]) return run def activate_run(run_id: uuid.UUID | str) -> PipelineRun: run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None: raise PipelineRunNotFoundError(uuid.UUID(str(run_id))) run.status = RunStatus.running run.started_at = timezone.now() run.save() return run def finalize_run( run_id: uuid.UUID | str, *, finished_reason: RunFinishedReason = RunFinishedReason.done, ) -> None: run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None: return run.status = RunStatus.done run.finished_at = timezone.now() run.finished_reason = finished_reason run.save() def pause_run(run_id: uuid.UUID | str) -> PipelineRun | None: run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None or run.status != RunStatus.running: return run run.status = RunStatus.paused run.save() return run def resume_run(run_id: uuid.UUID | str) -> PipelineRun | None: run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None or run.status != RunStatus.paused: return run run.status = RunStatus.running run.save() return run def cancel_run( run_id: uuid.UUID | str, *, reason: RunFinishedReason = RunFinishedReason.cancelled, ) -> PipelineRun | None: run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None or run.status not in ( RunStatus.queued, RunStatus.running, RunStatus.paused, ): return run run.status = RunStatus.cancelled run.finished_at = timezone.now() run.finished_reason = reason run.save() return run def stall_run(run_id: uuid.UUID | str) -> PipelineRun | None: """Manually mark an active run as stalled (error/stalled).""" run = PipelineRun.query.where(PipelineRun.id == run_id).first() if run is None or run.status not in ( RunStatus.queued, RunStatus.running, RunStatus.paused, ): return run run.status = RunStatus.error run.finished_at = timezone.now() run.finished_reason = RunFinishedReason.stalled run.save() return run