import logging import time from collections.abc import Sequence from concurrent.futures import ThreadPoolExecutor from datetime import datetime, timedelta from typing import Any from fansifter_common.adapters.twilio.exceptions import TwilioClientError from fansifter_common.adapters.twilio.models import SendMessageRequest from fansifter_common.adapters.twilio.utils import format_whatsapp_address from fansifter_common.utils import timezone from fansifter_common.utils.uuid import uuid_string from app import lock from app.adapters.kms import kms_client from app.adapters.ows_text_campaigns import ows_text_campaigns_client from app.adapters.postgres import postgres_db from app.adapters.snowflake import snowflake_db from app.adapters.twilio import twilio_client from app.config import settings from app.enums import StopReason from app.models import ( ArtistSettings, BatchRecipient, BatchSendRecord, Campaign, CampaignBatch, TwilioAccount, ) from app.types import ( FanSendOutcome, MessageChannel, PersonalizedAttributes, SendBatchRequest, TwilioSendCredentials, ) RETRY_BACKOFF_SECONDS = 0.2 logger = logging.getLogger(__name__) def handle(event: dict[str, Any]) -> None: request = SendBatchRequest.model_validate(event) log_extra = {"batch_id": request.batch_id, "campaign_id": request.campaign_id} with snowflake_db.session_factory(): batch = ( CampaignBatch.query.joinedload(CampaignBatch.campaign) .where(CampaignBatch.id == request.batch_id) .one_or_none() ) if batch is None or not batch.is_active: logger.warning( "Batch `%s` is not found or not active; likely already completed by " "another invocation. Nothing to send.", request.batch_id, extra=log_extra, ) lock.release(request.batch_id) return campaign = batch.campaign if campaign.is_cancelled or campaign.is_deleted: logger.info( "Campaign `%s` was cancelled or deleted; aborting.", request.campaign_id, extra=log_extra, ) lock.release(request.batch_id) return target_offset = min(batch.batch_size, batch.batch_offset + request.messages_to_send) budget = timezone.now() + timedelta(seconds=settings.sender_time_budget_seconds) deadline = ( budget if request.safe_until is None else min( budget, request.safe_until - timedelta(seconds=settings.safe_until_buffer_seconds), ) ) try: credentials = _resolve_credentials(campaign.global_participant_id) except Exception as exc: logger.error( "Failed to resolve Twilio send credentials for campaign `%s`; " "aborting this invoke, next tick will retry.", request.campaign_id, exc_info=exc, extra=log_extra, ) lock.release(request.batch_id) return if credentials is None: logger.error( "No Twilio account configured for campaign `%s`'s artist; aborting.", request.campaign_id, extra=log_extra, ) lock.release(request.batch_id) return try: _process_batch( batch_id=request.batch_id, campaign_id=request.campaign_id, target_offset=target_offset, deadline=deadline, credentials=credentials, ) finally: lock.release(request.batch_id) def _resolve_credentials(global_participant_id: str) -> TwilioSendCredentials | None: with snowflake_db.session_factory(): artist_settings = ArtistSettings.query.get(global_participant_id) if ( artist_settings is None or not artist_settings.twilio_account_sid or not artist_settings.twilio_messaging_service_sid ): return None with postgres_db.session_factory(): twilio_account = TwilioAccount.query.get(artist_settings.twilio_account_sid) if twilio_account is None: return None return TwilioSendCredentials( account_sid=artist_settings.twilio_account_sid, messaging_service_sid=artist_settings.twilio_messaging_service_sid, api_key_id=twilio_account.api_key_id, api_secret=kms_client.decrypt(twilio_account.api_secret), ) def _process_batch( *, batch_id: int, campaign_id: str, target_offset: int, deadline: datetime, credentials: TwilioSendCredentials, ) -> None: with ThreadPoolExecutor(max_workers=settings.sender_concurrency) as executor: while True: has_next_chunk = _process_next_chunk( batch_id=batch_id, campaign_id=campaign_id, target_offset=target_offset, deadline=deadline, credentials=credentials, executor=executor, ) if not has_next_chunk: break with snowflake_db.transaction(): fully_sent = Campaign.query.mark_sent_if_fully_delivered(campaign_id) if fully_sent: logger.info( "Campaign `%s` fully sent.", campaign_id, extra={"campaign_id": campaign_id} ) def _process_next_chunk( *, batch_id: int, campaign_id: str, target_offset: int, deadline: datetime, credentials: TwilioSendCredentials, executor: ThreadPoolExecutor, ) -> bool: """Process one chunk of the batch. Returns whether there may be more to do.""" with snowflake_db.session_factory(): batch = ( CampaignBatch.query.joinedload(CampaignBatch.campaign) .where(CampaignBatch.id == batch_id) .one_or_none() ) if batch is None or not batch.is_active: logger.info( "Stopping batch processing for batch `%s`: batch is no longer active.", batch_id, extra={"batch_id": batch_id, "campaign_id": campaign_id}, ) return False stop_reason = _stop_reason(batch, target_offset, deadline) if stop_reason is not None: logger.info( "Stopping batch processing for batch `%s`: %s.", batch_id, stop_reason, extra={"batch_id": batch_id, "campaign_id": campaign_id}, ) return False chunk_size = min(settings.sender_chunk_size, target_offset - batch.batch_offset) recipients = BatchRecipient.query.for_chunk( batch_id=batch_id, offset=batch.batch_offset, limit=chunk_size ) if not recipients: logger.warning( "Batch `%s` has no recipients left at offset `%s` but is not " "complete; stopping to avoid an infinite loop.", batch_id, batch.batch_offset, extra={"batch_id": batch_id, "campaign_id": campaign_id}, ) return False outcomes = _send_chunk( campaign_id=campaign_id, channel=batch.campaign.channel.as_literal(), recipients=recipients, credentials=credentials, executor=executor, ) sent_count = sum(1 for outcome in outcomes if outcome.sent) progress_recorded = _record_progress( batch_id=batch_id, campaign_id=campaign_id, expected_offset=batch.batch_offset, recipient_count=len(recipients), sent_count=sent_count, ) if not progress_recorded: logger.warning( "Stopping batch processing for batch `%s`: progress could not be " "recorded, another invocation is likely already processing it.", batch_id, extra={"batch_id": batch_id, "campaign_id": campaign_id}, ) return False logger.info( "Batch `%s` progress recorded: offset advanced by %d (%d sent).", batch_id, len(recipients), sent_count, extra={ "batch_id": batch_id, "campaign_id": campaign_id, "sent_count": sent_count, }, ) return True def _stop_reason( batch: CampaignBatch, target_offset: int, deadline: datetime ) -> StopReason | None: campaign = batch.campaign if campaign.is_cancelled or campaign.is_deleted: return StopReason.CAMPAIGN_CANCELLED_OR_DELETED if timezone.now() >= deadline: return StopReason.DEADLINE_REACHED if batch.batch_offset >= target_offset: return StopReason.TARGET_OFFSET_REACHED return None def _record_progress( *, batch_id: int, campaign_id: str, expected_offset: int, recipient_count: int, sent_count: int, ) -> bool: now = timezone.now() with snowflake_db.transaction(): current = CampaignBatch.query.get(batch_id, with_for_update=True) if current is None: return False if current.batch_offset != expected_offset: logger.warning( "Batch `%s` offset drifted from `%s` to `%s` before recording " "progress; skipping to avoid double-counting.", batch_id, expected_offset, current.batch_offset, extra={"batch_id": batch_id, "campaign_id": campaign_id}, ) return False current.batch_offset += recipient_count if current.first_sent_at is None: current.first_sent_at = now current.last_sent_at = now if current.batch_offset >= current.batch_size: current.completed_at = now current.save() record = BatchSendRecord( batch_id=batch_id, batch_offset=current.batch_offset, sent_count=sent_count, sent_at=now, ) record.save() return True def _send_chunk( *, campaign_id: str, channel: MessageChannel, recipients: Sequence[BatchRecipient], credentials: TwilioSendCredentials, executor: ThreadPoolExecutor, ) -> list[FanSendOutcome]: attributes: list[PersonalizedAttributes] = [ { "fan_id": recipient.fan_id, "phone_number": recipient.fan_phone_number, "channel": channel, } for recipient in recipients ] try: rendered = ows_text_campaigns_client.render_messages_batch( campaign_id=campaign_id, attributes=attributes ) except Exception as exc: logger.error( "Failed to render messages for campaign `%s`; treating this chunk as " "entirely failed. Offset still advances -- a rendering outage must not " "wedge the batch forever.", campaign_id, exc_info=exc, extra={"campaign_id": campaign_id}, ) return [ FanSendOutcome(fan_id=recipient.fan_id, sent=False, error_message=str(exc)) for recipient in recipients ] rendered_by_fan_id = {message.fan_id: message for message in rendered} def _send(recipient: BatchRecipient) -> FanSendOutcome: rendered_message = rendered_by_fan_id.get(recipient.fan_id) if rendered_message is None: logger.warning( "No rendered message returned for fan `%s`; skipping.", recipient.fan_id, extra={ "fan_id": recipient.fan_id, "batch_id": recipient.batch_id, "campaign_id": campaign_id, }, ) return FanSendOutcome( fan_id=recipient.fan_id, sent=False, error_message="not rendered" ) to = rendered_message.recipient from_ = rendered_message.sender if channel == "WHATSAPP": to = format_whatsapp_address(to) from_ = format_whatsapp_address(from_) message_request = SendMessageRequest( to=to, from_=from_, body=rendered_message.message, media_url=rendered_message.media_url, messaging_service_sid=credentials.messaging_service_sid, account_sid=credentials.account_sid, ) auth = (credentials.api_key_id, credentials.api_secret) attempts = settings.sender_message_max_retries + 1 last_error: Exception | None = None for attempt in range(attempts): try: if _should_send_for_real(rendered_message.recipient): message_id = twilio_client.send_message( message_request, auth=auth ).sid else: logger.info( "Not actually sending (sender_backend=log or phone not " "allow-listed) -- to=`%s` from=`%s`.", message_request.to, message_request.from_, extra={"to": message_request.to, "from": message_request.from_}, ) message_id = f"dummy-{uuid_string()}" return FanSendOutcome( fan_id=recipient.fan_id, sent=True, message_id=message_id, ) except Exception as exc: last_error = exc if attempt >= attempts - 1 or not _is_retryable_send_error(exc): break time.sleep(RETRY_BACKOFF_SECONDS) logger.error( "Failed to send message to fan `%s` after %d attempt(s); a few bad " "recipients must not block the rest of the chunk.", recipient.fan_id, attempts, exc_info=last_error, extra={ "fan_id": recipient.fan_id, "batch_id": recipient.batch_id, "campaign_id": campaign_id, }, ) return FanSendOutcome( fan_id=recipient.fan_id, sent=False, error_message=str(last_error) if last_error else None, ) return list(executor.map(_send, recipients)) def _should_send_for_real(to: str) -> bool: if settings.sender_backend != "twilio": return False return ( not settings.allowed_recipient_phone_numbers or to in settings.allowed_recipient_phone_numbers ) def _is_retryable_send_error(exc: Exception) -> bool: return ( isinstance(exc, TwilioClientError) and exc.response is not None and exc.response.status_code >= 500 )