import json from datetime import datetime from app import config from app.logger import log def get_latest_cdc_event(snowflake, playlist_id): log.info("Getting latest CDC event") # Get latest CDC event log from Snowflake DB data_frame = snowflake.execute_query('spotify_playlists_processing_log', playlist_id=playlist_id) if data_frame.empty: log.warning(f"No CDC events found for playlist {playlist_id}") return None cdc_event = data_frame.iloc[0] return cdc_event def get_latest_cdc_event_timestamp(snowflake, playlist_id): log.info("Getting latest CDC event timestamp") cdc_event = get_latest_cdc_event(snowflake, playlist_id) if cdc_event is None: raise ValueError(f"No CDC event found for playlist {playlist_id}") latest_cdc_timestamp = json.loads(cdc_event['UPDATE_SENT'])[0] return datetime.strptime(latest_cdc_timestamp.replace(" Z", " +0000"), "%a, %d %b %Y %H:%M:%S %z") PLAYLIST_ID_CHUNK_SIZE = 1000 def get_ingested_playlist_ids(snowflake, store_id, playlist_ids): log.info(f"Getting playlist ids ingested for store {store_id} from v_playlist_metadata") ingested_ids = set() for i in range(0, len(playlist_ids), PLAYLIST_ID_CHUNK_SIZE): chunk = playlist_ids[i:i + PLAYLIST_ID_CHUNK_SIZE] placeholders = ", ".join(["%s"] * len(chunk)) data_frame = snowflake.execute_query( 'playlist_metadata_ids', params=chunk, store_id=str(store_id), placeholders=placeholders, env=config.ENV) # Normalize to str since callers diff this set against plain CSV-loaded strings. ingested_ids.update(str(pid).strip() for pid in data_frame['STORE_PLAYLIST_ID']) return ingested_ids