import csv import io import itertools import logging from typing import Any from fansifter_common.adapters.sendgrid.exceptions import SendGridClientError from fansifter_common.adapters.sendgrid.models import Email, FanData from fansifter_common.adapters.stripo.exceptions import StripoClientError from fansifter_common.constants import QA_ENVIRONMENT from fansifter_common.core.enums import Brand from fansifter_common.email import EmailType from fansifter_common.utils import timezone from unidecode import unidecode from app import dedup, fandata from app.adapters.aws import s3_client from app.adapters.db import db from app.adapters.ows import ows_account_client from app.adapters.sendgrid import get_error_log_msg, sendgrid_client from app.adapters.stripo import stripo_client from app.config import settings from app.exceptions import EmailSendError from app.models import AutomatedEmail, AutomatedEmailSendBatch, Domain from app.types import CsvRow, EmailSend, S3Event logger = logging.getLogger(__name__) MAX_EMAILS_TO_SEND = 1000 def handle(event: Any) -> None: s3_event = S3Event.model_validate(event) logger.info("Parsed S3 event.", extra={"event": s3_event.model_dump()}) bucket = s3_event.records[0].s3.bucket.name key = s3_event.records[0].s3.object.key logger.info( "Processing automated emails CSV.", extra={ "bucket": bucket, "key": key, }, ) content = s3_client.get_object_content(bucket, key=key) rows = _parse_csv(content) if not rows: logger.info( "CSV file is empty, nothing to send.", extra={ "key": key, }, ) return # CSV rows are expected to be pre-sorted so that # groupby correctly groups all rows for the same automated email together. for automated_email_id, group_iter in itertools.groupby( rows, key=lambda r: r.automated_email_id ): group_rows = list(group_iter) automated_email_trigger_id = group_rows[0].automated_email_trigger_id confirmation_id = group_rows[0].confirmation_id batch = _get_or_create_batch(key, automated_email_id) if batch.is_completed: logger.info( "Batch already completed, skipping.", extra={ "key": key, "automated_email_id": automated_email_id, }, ) continue email_send = _build_email_send( automated_email_id=automated_email_id, automated_email_trigger_id=automated_email_trigger_id, confirmation_id=confirmation_id, ) rows_to_process = [r for r in group_rows if r.row_id > batch.last_sent_row_id] max_row_id = max(r.row_id for r in group_rows) if not rows_to_process: with db.transaction(): batch.completed_at = timezone.now() batch.save() logger.info( "All rows already processed, marking batch as completed.", extra={ "key": key, "automated_email_id": automated_email_id, }, ) continue logger.info( "Processing %d rows.", len(rows_to_process), extra={ "key": key, "automated_email_id": automated_email_id, "last_sent_row_id": batch.last_sent_row_id, "total_rows": len(group_rows), }, ) _send_rows( batch=batch, rows=rows_to_process, email_send=email_send, max_row_id=max_row_id, key=key, ) logger.info( "Finished processing group.", extra={ "key": key, "automated_email_id": automated_email_id, "is_completed": batch.is_completed, }, ) def _build_email_send( automated_email_id: str, automated_email_trigger_id: str, confirmation_id: str | None, ) -> EmailSend: with db.session_factory(): automated_email = AutomatedEmail.query.get(automated_email_id) if automated_email is None: raise EmailSendError(f"AutomatedEmail {automated_email_id} not found.") if not automated_email.sender_name or not automated_email.email_username: raise EmailSendError("Sender details are missing.") if ( not automated_email.subject or not automated_email.html_content or not automated_email.css_content ): raise EmailSendError("Email content is missing.") if automated_email.domain_id is None: raise EmailSendError("Email domain is not set.") domain = Domain.query.get(automated_email.domain_id) if domain is None: raise EmailSendError("Email domain not found.") if automated_email.compressed_content: html_content = automated_email.compressed_content else: try: html_content = stripo_client.compress( html=automated_email.html_content, css=automated_email.css_content, ) except StripoClientError as exc: raise EmailSendError( f"AutomatedEmail {automated_email.id}: Failed to compress HTML content." ) from exc vendor = ows_account_client.get_vendor(automated_email.vendor_id) return EmailSend( automated_email_id=automated_email.id, automated_email_trigger_id=automated_email_trigger_id, brand=Brand(vendor.brand), vendor_id=vendor.vendor_id, vendor_name=vendor.name, subject=automated_email.subject, html_content=html_content, preview_text=automated_email.preview_text, from_email=Email( name=automated_email.sender_name, email=f"{automated_email.email_username}@{domain.domain}", ), category=unidecode(automated_email.sender_name), confirmation_id=confirmation_id, ) def _get_or_create_batch(key: str, automated_email_id: str) -> AutomatedEmailSendBatch: with db.transaction(): batch = AutomatedEmailSendBatch.query.where( AutomatedEmailSendBatch.automated_email_id == automated_email_id, AutomatedEmailSendBatch.key == key, ).first() if batch is None: batch = AutomatedEmailSendBatch( key=key, automated_email_id=automated_email_id, ) batch.save() return batch def _send_rows( *, batch: AutomatedEmailSendBatch, rows: list[CsvRow], email_send: EmailSend, max_row_id: int, key: str, ) -> None: for fans in itertools.batched(rows, MAX_EMAILS_TO_SEND, strict=False): fan_data_list = [ fan_data for row in fans if (fan_data := fandata.get_fan_data(row, email_send.automated_email_id)) is not None ] batch_last_row_id = fans[-1].row_id fan_data_list = dedup.filter_unsent( automated_email_id=email_send.automated_email_id, fan_data_list=fan_data_list, ) emails_sent = False if fan_data_list and not _should_skip_email_sending(email_send): _send_via_sendgrid( fan_data_list=fan_data_list, email_send=email_send, key=key, ) dedup.mark_sent(email_send.automated_email_id, fan_data_list) emails_sent = True with db.transaction(): batch.last_sent_row_id = batch_last_row_id if batch_last_row_id >= max_row_id: batch.completed_at = timezone.now() batch.save() if emails_sent: AutomatedEmail.query.update_last_sent_at(email_send.automated_email_id) def _send_via_sendgrid( *, fan_data_list: list[FanData], email_send: EmailSend, key: str ) -> None: try: sendgrid_client.send_mail( email_id=email_send.automated_email_id, email_type=EmailType.AUTOMATED, automated_email_trigger_id=email_send.automated_email_trigger_id, version="v3", brand=email_send.brand, vendor_id=email_send.vendor_id, vendor_name=email_send.vendor_name, from_email=email_send.from_email, subject=email_send.subject, html_content=email_send.html_content, to_fans=fan_data_list, preview_text=email_send.preview_text, category=email_send.category, ) logger.info( "Sent %d emails via SendGrid.", len(fan_data_list), extra={ "automated_email_id": email_send.automated_email_id, "key": key, }, ) except SendGridClientError as exc: logger.error( get_error_log_msg(exc), exc_info=exc, extra={ "automated_email_id": email_send.automated_email_id, "key": key, **exc.log_extra, }, ) raise EmailSendError("Failed to send emails via SendGrid.") from exc def _parse_csv(content: str) -> list[CsvRow]: rows: list[CsvRow] = [] reader = csv.DictReader(io.StringIO(content)) for row in reader: try: rows.append( CsvRow( row_id=int(row["row_id"]), automated_email_id=row["automated_email_id"], automated_email_trigger_id=row["automated_email_trigger_id"], timestamp=row["timestamp"], is_doi_send=row["is_doi_send"].lower() == "true", confirmation_id=row["confirmation_id"] or None, fan_profile_id=row["fan_profile_id"] or None, fan_email=row["fan_email"], fan_first_name=row["fan_first_name"], fan_last_name=row["fan_last_name"], fan_country_code=row["fan_country_code"], ) ) except (KeyError, ValueError) as exc: logger.error( "Skipping malformed CSV row: %s", exc, extra={"row": dict(row)} ) return rows def _should_skip_email_sending(email_send: EmailSend) -> bool: return ( settings.environment == QA_ENVIRONMENT and settings.send_skip_tag in email_send.subject )