"""Lambda entry point for spotify_daily_chart_freshness. Triggered by EventBridge every 5 minutes. Detects when Spotify releases a new daily chart date and alerts if FACT_CHARTS hasn't caught up within LAG_ALERT_MINUTES (default: 15). State machine (persisted in SSM): IDLE — last_fact_date == yesterday; Spotify and Snowflake are NOT queried PENDING — new Spotify date detected; Snowflake queried each run until resolved ALERTED — lag exceeded threshold; alert emitted once, Snowflake still checked for auto-resolve """ from datetime import date, datetime, timedelta, timezone from datadog_lambda.metric import lambda_metric from src import spotify, snowflake, state import config logger = config.logger def handler(event, context): """Lambda entry point.""" try: _run() except Exception: logger.exception('Unhandled error in spotify_daily_chart_freshness') raise def _run(): current_state = state.load() # --- Step 0: Short-circuit if FACT_CHARTS already has yesterday's date --- # No need to hit Spotify or Snowflake — pipeline is current. # Condition expires naturally at midnight when 'yesterday' rolls forward. last_fact_date = current_state.get('last_fact_date') today = date.today() yesterday = today - timedelta(days=1) if last_fact_date: try: parsed = date.fromisoformat(last_fact_date) if yesterday <= parsed <= today: logger.info( f'FACT_CHARTS already at {last_fact_date}. Skipping checks.' ) # Spotify not called here — omit spotify_up to avoid masking outages lambda_metric(config.DD_LAG_METRIC, 0, tags=config.DD_TAGS) lambda_metric(config.DD_ALERT_METRIC, 0, tags=config.DD_TAGS) return if parsed > today: logger.warning( f'last_fact_date {last_fact_date!r} is a future date; ' 'proceeding with checks' ) except ValueError: logger.warning( f'Invalid last_fact_date {last_fact_date!r} in state; ' 'proceeding with checks' ) # --- Step 1: Check Spotify provider API --- spotify_date = spotify.get_latest_chart_date() spotify_up = spotify_date is not None lambda_metric( config.DD_SPOTIFY_UP_METRIC, 1 if spotify_up else 0, tags=config.DD_TAGS ) pending_date = current_state.get('pending_date') if not spotify_up: if not pending_date: # No active pending date and Spotify down — nothing to track logger.warning('Spotify API unavailable and no pending date; skipping') lambda_metric(config.DD_LAG_METRIC, 0, tags=config.DD_TAGS) lambda_metric(config.DD_ALERT_METRIC, 0, tags=config.DD_TAGS) return # Pending date already in state — skip date detection but still check # Snowflake so the monitor can auto-resolve during a Spotify outage logger.warning( f'Spotify API unavailable; continuing Snowflake check for {pending_date}' ) else: # --- Step 2: Detect new chart date --- now = datetime.now(timezone.utc) if pending_date != spotify_date: logger.info( f'New Spotify daily chart date detected: {spotify_date}' f' (previous: {pending_date})' ) current_state = { 'pending_date': spotify_date, 'detected_at': now.isoformat(), 'alerted': False, } state.save(current_state) pending_date = spotify_date # --- Step 3: Query Snowflake (only when a new date is pending) --- now = datetime.now(timezone.utc) try: detected_at = datetime.fromisoformat(current_state['detected_at']) if detected_at.tzinfo is None: # Normalize naive timestamps to UTC and persist so arithmetic works detected_at = detected_at.replace(tzinfo=timezone.utc) current_state = {**current_state, 'detected_at': detected_at.isoformat()} state.save(current_state) if detected_at > now: # Future detected_at produces negative lag — clamp to now detected_at = now current_state = {**current_state, 'detected_at': now.isoformat()} state.save(current_state) except (KeyError, ValueError, TypeError): # Corrupt or missing detected_at — repair and persist so lag accumulates # correctly on subsequent invocations and can eventually trigger an alert. detected_at = now current_state = {**current_state, 'detected_at': now.isoformat()} state.save(current_state) fact_date = snowflake.get_latest_fact_chart_date() if fact_date == pending_date: lag_minutes = (now - detected_at).total_seconds() / 60 logger.info( f'FACT_CHARTS in sync at {pending_date} (lag was {lag_minutes:.1f} min).' ) state.save({'last_fact_date': pending_date}) lambda_metric(config.DD_LAG_METRIC, 0, tags=config.DD_TAGS) lambda_metric(config.DD_ALERT_METRIC, 0, tags=config.DD_TAGS) return # --- Step 4: Compute lag and decide whether to alert --- lag_minutes = (now - detected_at).total_seconds() / 60 lambda_metric(config.DD_LAG_METRIC, lag_minutes, tags=config.DD_TAGS) logger.info( f'Chart lag: {lag_minutes:.1f} min | ' f'Spotify={pending_date} | FACT_CHARTS={fact_date}' ) already_alerted = current_state.get('alerted', False) if lag_minutes > config.LAG_ALERT_MINUTES: if not already_alerted: logger.error( 'SPOTIFY CHART FRESHNESS ALERT: ' f'Spotify has chart date {pending_date} ' f'since {detected_at.isoformat()} ' f'({lag_minutes:.1f} min ago) but FACT_CHARTS is still at {fact_date}. ' 'Likely Chartmetric pipeline issue or downstream system error.' ) state.save({**current_state, 'alerted': True}) # Always emit 1 while alert is active — avoids metric time-series gaps lambda_metric(config.DD_ALERT_METRIC, 1, tags=config.DD_TAGS) else: lambda_metric(config.DD_ALERT_METRIC, 0, tags=config.DD_TAGS)