import os import sys import traceback from datetime import datetime, timedelta from typing import Dict, List, Set import boto3 import redis import sentry_sdk from apollo_notifications.decorators import loop, time_logger from apollo_notifications.main_db import session_scope from apollo_notifications.playlists import PushTrackInPlaylistSchema, StarredSpotifyPlaylistEntryPushSchema from apollo_notifications.push_client import Push, PushClient, PushData from apollo_notifications.push_client.exceptions import PushClientError from apollo_notifications.redis import ConstantKeyCache, lock from apollo_notifications.user_data.client import UserDataClient from ddtrace import patch_all, tracer from sentry_sdk.utils import BadDsn from sqlalchemy.exc import SQLAlchemyError from client import SpotifyPlaylistClient from config import config from logger import logger if os.environ.get("DATADOG_SERVICE_NAME"): patch_all() else: tracer.enabled = False try: if config.ENVIRONMENT != "local": sentry_sdk.init(dsn=config.SENTRY_DSN, environment=config.ENVIRONMENT) except BadDsn: pass redis_client = redis.StrictRedis(config.REDIS_HOST, config.REDIS_PORT) def process_starred_playlists_entries_by_market( count: int, market: str, today: str, yesterday: str, include_playlist_id_list: List[str], user_id_list: List[str], include_playlist_to_users_map: Dict[str, Set[str]], existing_messages: Set[str], notifications_client: SpotifyPlaylistClient, push_client: PushClient, users_map: Dict[str, int] = None, ): starred_playlist_additions_schema = StarredSpotifyPlaylistEntryPushSchema( title=config.STARRED_PLAYLIST_ADDITION_TITLE, topic=config.STARRED_PLAYLIST_ADDITION_TOPIC, vendor=config.VENDOR, message_template=config.STARRED_PLAYLIST_ADDITION_MESSAGE_TEMPLATE, date=today, market=market, push_cls=Push, push_data_cls=PushData, users_map=users_map, ) starred_playlists_for_market = notifications_client.get_updated_playlists_by_date_query( market, today, filter_by_playlists=include_playlist_id_list ) tracks_added_to_starred_playlists_query = notifications_client.get_added_tracks_query( today, yesterday, starred_playlists_for_market, starred_playlists_for_market ) filtered_tracks_added_to_starred_playlists_query = notifications_client.filter_by_starred_and_active( tracks_added_to_starred_playlists_query, user_id_list ) starred_playlists_messages = notifications_client.get_push_messages( query=filtered_tracks_added_to_starred_playlists_query, push_date=today, markets=[market], # we don't send notification for starred playlist if user has already received top playlist notification # with the same parameters (case when starring top playlist after getting top playlist notification) topics=[config.STARRED_PLAYLIST_ADDITION_TOPIC, config.TOPIC], push_schema=starred_playlist_additions_schema, existing_messages=existing_messages, include_playlist_id_to_users_map=include_playlist_to_users_map, ) push_client.process_messages(starred_playlists_messages) logger.info( f"2.{count}.1 Created {len(starred_playlists_messages)} starred playlists messages " f"for {market} market." ) def process_top_playlists_entries_by_market( count: int, market: str, today: str, yesterday: str, excluded_playlist_id_list: List[str], user_id_list: List[str], exclude_playlist_to_users_map: Dict[str, Set[str]], existing_messages: Set[str], notifications_client: SpotifyPlaylistClient, push_client: PushClient, users_map: Dict[str, int] = None, ): top_playlist_additions_schema = PushTrackInPlaylistSchema( title=config.TITLE, topic=config.TOPIC, vendor=config.VENDOR, message_template=config.MESSAGE_TEMPLATE, date=today, market=market, push_cls=Push, push_data_cls=PushData, users_map=users_map, ) top_playlist_ids_for_today = notifications_client.get_top_playlists_by_date_query( market, today, exclude=excluded_playlist_id_list ) top_playlist_ids_for_yesterday = notifications_client.get_top_playlists_by_date_query( market, yesterday, exclude=excluded_playlist_id_list ) tracks_added_to_top_playlists_query = notifications_client.get_added_tracks_query( today, yesterday, top_playlist_ids_for_today, top_playlist_ids_for_yesterday ) filtered_tracks_added_to_top_playlists_query = notifications_client.filter_by_starred_and_active( tracks_added_to_top_playlists_query, user_id_list ) top_playlists_messages = notifications_client.get_push_messages( query=filtered_tracks_added_to_top_playlists_query, push_date=today, markets=[market], # we don't send notification for top playlist if user has already received starred playlist notification # with the same params: case when adding playlist market to settings after getting starred playlist notification topics=[config.STARRED_PLAYLIST_ADDITION_TOPIC, config.TOPIC], push_schema=top_playlist_additions_schema, existing_messages=existing_messages, exclude_playlist_id_to_users_map=exclude_playlist_to_users_map, ) push_client.process_messages(top_playlists_messages) logger.info(f"2.{count}.2 Created {len(top_playlists_messages)} top playlists messages for {market} market.") @loop(logger, config) def job( notifications_client: SpotifyPlaylistClient, push_client: PushClient, user_data_client: UserDataClient, blacklist_cache: ConstantKeyCache, ): # '_job' is separated from 'job' to allow its testing without wrapping decorators return _job( notifications_client=notifications_client, push_client=push_client, user_data_client=user_data_client, blacklist_cache=blacklist_cache, ) def _job( notifications_client: SpotifyPlaylistClient, push_client: PushClient, user_data_client: UserDataClient, blacklist_cache: ConstantKeyCache, ): today = datetime.today().strftime("%Y-%m-%d") yesterday = (datetime.today().date() - timedelta(days=1)).strftime("%Y-%m-%d") logger.info(f"0. Started processing update for dates: {today} and {yesterday}.") markets_to_users_map, users_map = user_data_client.parse_v1_mobile_settings(config.VENDOR, with_account_id=True) users_list = list(users_map.keys()) logger.info( f"1.0. Got {len(users_list)} users to send starred and top playlist additions notifications for: {users_list}." ) starred_playlists_to_users_map, starred_playlists_id_set = user_data_client.parse_starred_playlists( config.VENDOR, users_list ) with session_scope(): blacklisted_ids = set(blacklist_cache(notifications_client.get_blacklisted_ids)()) logger.info(f"1.1. Got {len(blacklisted_ids)} blacklisted playlists: {blacklisted_ids}.") starred_playlists_id_list = list(starred_playlists_id_set - blacklisted_ids) logger.info( f"1.2. Got {len(starred_playlists_id_list)} starred playlists to create notifications " f"for: {starred_playlists_id_list}." ) top_markets = ( notifications_client.get_top_markets(filter_by_markets=markets_to_users_map.keys()) if markets_to_users_map else set() ) logger.info(f"1.3.1 Got {len(top_markets)} top markets: {top_markets}.") starred_playlists_markets = ( notifications_client.get_updated_tracklist_markets( target_date=today, filter_by_playlists=starred_playlists_id_list ) if starred_playlists_id_list else set() ) logger.info( f"1.3.2 Got {len(starred_playlists_markets)} starred playlists markets: {starred_playlists_markets}." ) markets = top_markets | starred_playlists_markets existing_messages = notifications_client.get_existing_push_messages( target_push_date=today, topics=[config.TOPIC, config.STARRED_PLAYLIST_ADDITION_TOPIC] ) logger.info( f"1.4. Got {len(existing_messages)} existing push messages for " f"{today} + {config.TOPIC}/{config.STARRED_PLAYLIST_ADDITION_TOPIC} + {config.VENDOR}" ) for i, market in enumerate(markets): try: with session_scope() as session: push_client.set_db_session(session) if market in starred_playlists_markets: process_starred_playlists_entries_by_market( count=i, market=market, today=today, yesterday=yesterday, include_playlist_id_list=starred_playlists_id_list, user_id_list=users_list, # we do not filter starred playlists messages by settings markets include_playlist_to_users_map=starred_playlists_to_users_map, existing_messages=existing_messages, notifications_client=notifications_client, push_client=push_client, users_map=users_map, ) if market in markets_to_users_map: process_top_playlists_entries_by_market( count=i, market=market, today=today, yesterday=yesterday, excluded_playlist_id_list=blacklisted_ids, user_id_list=list(markets_to_users_map[market]), # we do filter top playlists messages # by settings markets exclude_playlist_to_users_map=starred_playlists_to_users_map, # do not send for top if push # was already sent for starred existing_messages=existing_messages, notifications_client=notifications_client, push_client=push_client, users_map=users_map, ) except (SQLAlchemyError, PushClientError) as ex: sentry_sdk.capture_exception(ex) logger.error(traceback.format_exception(*sys.exc_info()) + traceback.format_stack()) finally: markets_to_users_map.pop(market, None) @time_logger(logger, config.APP_NAME) @lock(logger, config, redis_client, sentry=sentry_sdk) def handler(): """Save push messages to db and send them to queue.""" sqs_client = boto3.client("sqs") user_data_client = UserDataClient(config, logger) notifications_client = SpotifyPlaylistClient(config) push_client = PushClient(logger=logger, sqs_client=sqs_client, config=config) blacklist_cache = ConstantKeyCache(redis_client, config.BLACKLIST_KEY, config.BLACKLIST_TTL) job(notifications_client, push_client, user_data_client, blacklist_cache) if __name__ == "__main__": handler()