"""Fan dispatcher — reads fan_credentials for a run and dispatches FanBatches.""" import logging import math import uuid from dataclasses import dataclass from app.adapters.db import db from app.config import settings from app.dsp.enums import DSPClientName from app.dsp.models import DSPClient from app.fandata.models import FanCredentials, FanCredentialsFilter from app.fandata.types import FanBatch, FanRecord from app.pipeline import services from app.pipeline.dispatch import dispatch_fan_collect from app.pipeline.enums import RunFinishedReason from app.pipeline.exceptions import ( PipelineAlreadyRunningError, PipelineRunNotFoundError, ) logger = logging.getLogger(__name__) @dataclass(frozen=True) class _RunContext: run_id: uuid.UUID client_name: DSPClientName batch_size: int total_fans: int total_batches: int filters: FanCredentialsFilter def fanout(run_id: uuid.UUID) -> None: """Read fan_credentials for the queued run and dispatch FanBatches to the broker. Activation (queued → running) is committed before any tasks are dispatched, so workers always see status=running when they pick up a batch. """ with db.transaction(): run_context = _activate_and_prepare(run_id) if run_context is None: return cursor_id = 0 dispatched = 0 while dispatched < run_context.total_fans: remaining = run_context.total_fans - dispatched with db.autocommit(): page = FanCredentials.query.filter(run_context.filters).cursor_page( after_id=cursor_id, limit=min(run_context.batch_size, remaining), ) if not page: break fans = [ FanRecord( dsp_user_id=cred.dsp_user_id, refresh_token_encrypted=cred.refresh_token_encrypted, ) for cred in page ] dispatch_fan_collect( FanBatch(run_id=run_context.run_id, fans=fans), run_context.client_name, ) cursor_id = page[-1].id dispatched += len(page) logger.info( "Dispatch done: run_id=%s total_fans=%d total_batches=%d", run_context.run_id, run_context.total_fans, run_context.total_batches, extra={ "run_id": str(run_context.run_id), "dsp_client": run_context.client_name, "total_fans": run_context.total_fans, "total_batches": run_context.total_batches, }, ) def _activate_and_prepare(run_id: uuid.UUID) -> _RunContext | None: """Activate the run (queued → running) and count fans. Commits before returning.""" try: run = services.activate_run(run_id) except (PipelineAlreadyRunningError, PipelineRunNotFoundError) as exc: logger.warning( "Run %s — cannot start, dropping: %s", run_id, exc, extra={"run_id": str(run_id)}, ) return None client = DSPClient.query.where(DSPClient.id == run.dsp_client_id).one_or_none() if client is None: logger.error( "Run %s — dsp client %d not found, dropping", run_id, run.dsp_client_id, extra={"run_id": str(run_id)}, ) services.finalize_run(run.id, finished_reason=RunFinishedReason.done) return None filters = run.filters or FanCredentialsFilter(dsp_client_id=run.dsp_client_id) batch_size = settings.fan_fanout_batch_size_for(client.name) run.total_fans = FanCredentials.query.filter(filters).count() if filters.limit is not None: run.total_fans = min(run.total_fans, filters.limit) run.total_batches = ( math.ceil(run.total_fans / batch_size) if run.total_fans > 0 else 0 ) run.save() if run.total_fans == 0: logger.info( "Run %s — no credentials found, cancelling", run.id, extra={"run_id": str(run.id), "dsp_client": client.name}, ) services.cancel_run(run.id, reason=RunFinishedReason.no_credentials) return None logger.info( "Run started: run_id=%s client=%s fans=%d batches=%d", run.id, client.name, run.total_fans, run.total_batches, extra={ "run_id": str(run.id), "dsp_client": client.name, "total_fans": run.total_fans, "total_batches": run.total_batches, }, ) return _RunContext( run_id=run.id, client_name=client.name, batch_size=batch_size, total_fans=run.total_fans, total_batches=run.total_batches, filters=filters, )