import logging import uuid from datetime import datetime, timedelta from typing import List from apollo_push_client import Push, PushClient, PushData from botocore.client import BaseClient from redis import StrictRedis from sqlalchemy import and_, func from sqlalchemy.orm import Query from sqlalchemy.orm.session import Session from apollo_main_db.apollo.models import ApolloMarket, StarredContent, UserMarket from apollo_main_db.push_notifications.models import TopicEnum, UserDeviceToken, VendorEnum from apollo_main_db.spotify.models import SpotifyArtist, SpotifyPlaylist, SpotifyPlaylistTrackListHistory2, \ SpotifyTrack2, SpotifyTrackArtist, SpotifyWeeklyTopPlaylist from spotify_playlists_additions_notifications import constants from spotify_playlists_additions_notifications.config import Config from spotify_playlists_additions_notifications.main_db import session_scope from spotify_playlists_additions_notifications.redis_db import get_key from spotify_playlists_additions_notifications.utils import build_push_message, is_new_track, get_blacklisted_ids, \ get_country_code, get_country_flag def get_top_playlists_by_date(session: Session, market: str, date: str) -> List[bytes]: """Get top playlist by date Args: session (Session): DB session. market (str): Market. date (str): Date. Returns: List of playlist_id """ max_date_query = session.query( func.max(SpotifyWeeklyTopPlaylist.date) ).filter(SpotifyWeeklyTopPlaylist.date <= date) top_playlists = session.query( SpotifyWeeklyTopPlaylist.playlist_id, ).filter( SpotifyWeeklyTopPlaylist.country == market, SpotifyWeeklyTopPlaylist.rank <= constants.TOP_PLAYLIST_THRESHOLD, SpotifyWeeklyTopPlaylist.date == max_date_query, SpotifyWeeklyTopPlaylist.playlist_id.isnot(None) ) # FIXME: For some reason SpotifyPlaylistTrackListHistory2.playlist_id has # ascii charset, so we should encode our string to ascii. encoded_top_playlists = [] for playlist_id, in top_playlists: try: encoded_top_playlists.append(playlist_id.encode('ascii')) except UnicodeEncodeError: continue return encoded_top_playlists def get_added_tracks_query(session: Session, date: str, previous_date: str, market: str) -> Query: """Get added tracks to top playlists between two dates Args: session (Session): DB session. date (str): Current date for track history. previous_date (str): Previous date for track history. market (str): Market. Returns: List of tuple(playlist_uri, track_id,) """ top_playlists_for_date = get_top_playlists_by_date(session, market, date) top_playlists_for_prev_date = get_top_playlists_by_date(session, market, previous_date) end_date_tracks_query = session.query( SpotifyPlaylistTrackListHistory2.playlist_id.label('playlist_id'), SpotifyPlaylistTrackListHistory2.track_id.label('track_id'), ).filter( SpotifyPlaylistTrackListHistory2.playlist_id.in_( top_playlists_for_prev_date ), SpotifyPlaylistTrackListHistory2.date == previous_date ).subquery() start_date_tracks_query = session.query( SpotifyPlaylistTrackListHistory2.playlist_id.label('playlist_id'), (SpotifyPlaylistTrackListHistory2.playlist_index + 1).label('position'), SpotifyPlaylistTrackListHistory2.track_id.label('track_id'), SpotifyTrack2.name.label('track_name'), SpotifyTrack2.isrc.label('isrc'), SpotifyArtist.name.label('artist_name'), SpotifyPlaylist.name.label('playlist_name'), ).filter( SpotifyPlaylistTrackListHistory2.playlist_id.in_( top_playlists_for_date ), SpotifyPlaylistTrackListHistory2.date == date, SpotifyTrackArtist.order == 0, ).join( SpotifyTrackArtist, SpotifyTrackArtist.track_id == SpotifyPlaylistTrackListHistory2.track_id ).join( SpotifyArtist, SpotifyArtist.id == SpotifyTrackArtist.artist_id ).join( SpotifyTrack2, SpotifyTrack2.id == SpotifyPlaylistTrackListHistory2.track_id ).join( SpotifyPlaylist, SpotifyPlaylist.id == SpotifyPlaylistTrackListHistory2.playlist_id ) return start_date_tracks_query.outerjoin( end_date_tracks_query, and_( SpotifyPlaylistTrackListHistory2.playlist_id == end_date_tracks_query.c.playlist_id, SpotifyPlaylistTrackListHistory2.track_id == end_date_tracks_query.c.track_id ) ).filter(end_date_tracks_query.c.track_id.is_(None)) def get_only_starred_tracks_with_device_tokens(session: Session, tracks_sub_query: Query, market: str) -> Query: """Filtering tracks query by starred tracks and join with User devices Args: session (Session): DB session. tracks_sub_query: Query with track_id, position, track_name, artist_name, playlist_name, playlist_id aliased columns. market: str market. """ return session.query( StarredContent.user_id, ApolloMarket.code.label("market_name"), func.group_concat( UserDeviceToken.token.op('ORDER BY')(UserDeviceToken.created_at.desc()) ).label('device_tokens'), tracks_sub_query ).join( tracks_sub_query, func.upper(StarredContent.isrc) == func.upper(tracks_sub_query.c.isrc) ).join( UserDeviceToken, UserDeviceToken.user_id == StarredContent.user_id ).join( UserMarket, UserMarket.user_id == StarredContent.user_id ).join( ApolloMarket, ApolloMarket.id == UserMarket.market_id ).filter( ApolloMarket.code == market, UserDeviceToken.is_active.is_(True) ).group_by( StarredContent.user_id, tracks_sub_query.c.playlist_id, tracks_sub_query.c.track_id ) def process_added_tracks(market: str, today: str, yesterday: str, push_client: PushClient, redis_client: StrictRedis, session: Session, logger: logging.Logger, blacklisted_ids: list): """Send push notifications about added tracks. Args: market (str): Market code. today (str): Current date. yesterday (str): Previous date. push_client (PushClient): Push client instance. redis_client (StrictRedis): Redis client. session (Session): DB session. logger (Logger): Logging instance. """ added_tracks_query = get_added_tracks_query(session, today, yesterday, market).subquery() if market.lower() == constants.SPOTIFY_GLOBAL_MARKET: market = constants.APOLLO_GLOBAL_MARKET starred_tracks = get_only_starred_tracks_with_device_tokens(session, added_tracks_query, market=market) push_messages = [] for t in starred_tracks: _is_blacklisted = bool(blacklisted_ids and str(t.playlist_id) in blacklisted_ids) if _is_blacklisted is True: logger.info(f'Track <{t.track_id}> is skipped! <{t.playlist_id}> is in blacklist.') continue _is_new_track = is_new_track(today, t.playlist_id, t.track_id, market, t.user_id, redis_client) if _is_new_track is False: continue _country_code = get_country_code(market) _message_id = str(uuid.uuid4()) push_message = Push( id=_message_id, title='Added to a Top 250 Playlist Spotify', message=build_push_message(track=t, country_flag=get_country_flag(_country_code)), tokens=t.device_tokens.split(','), user_id=t.user_id, data=PushData( id=_message_id, isrc=t.isrc, country_code=_country_code, artist_name=t.artist_name, track_id=t.track_id, track_name=t.track_name, topic=TopicEnum.PLAYLIST_ADDITIONS.name.lower(), position=t.position, target=t.playlist_name, vendor=VendorEnum.SPOTIFY.name.lower(), playlist_id=t.playlist_id ) ) push_messages.append(push_message) for sent_messages in push_client.send_messages(push_messages): push_client.save_messages_to_db(sent_messages) def handler(logger: logging.Logger, sqs_client: BaseClient, redis_client: StrictRedis, app_config: Config): """Job for fetching new tracks in top Spotify playlists. Args: logger: Logger instance. sqs_client: Client for SQS AWS. redis_client: Redis client. app_config: Config object. Returns: JSON serializable response """ today = datetime.today().strftime("%Y-%m-%d") yesterday = (datetime.today().date() - timedelta(days=1)).strftime("%Y-%m-%d") # set lock for 15 m to avoid possible race condition between lambdas lock = redis_client.lock(get_key('lock'), timeout=constants.REDIS_LOCK_TTL) if not lock.acquire(blocking=False): logger.info('Locked') return {} with session_scope() as session: country_codes = session.query(SpotifyWeeklyTopPlaylist.country).distinct(SpotifyWeeklyTopPlaylist.country).all() blacklisted_ids = get_blacklisted_ids(session, redis_client) or [] logger.info(f"Blacklist: {blacklisted_ids}") for country_code, in country_codes: with session_scope() as session: push_client = PushClient( logger_client=logger, sqs_client=sqs_client, db_session=session, config=app_config) process_added_tracks(country_code, today, yesterday, push_client, redis_client, session, logger, blacklisted_ids) lock.release() return {}