from datetime import date, datetime, time, timedelta from typing import List, Optional, Tuple from apollo_main_db.spotify.models import SpotifyNewMusicFridayPlaylistTrackHistory import config import dsp_api import eventbridge import main_db import redis_db from json_logger import logger, trace_id def get_date(date_str: Optional[str]) -> Optional[datetime]: if not date_str: return None if "." in date_str and len(date_str) < 26: date_str += "0" * (26 - len(date_str)) return datetime.fromisoformat(date_str) def get_friday() -> date: """Get friday date. Returns: Friday. """ friday = datetime.utcnow().date() weekday = friday.weekday() if weekday == 3: friday += timedelta(days=1) elif weekday > 4: friday += timedelta(days=(4 - weekday)) elif weekday < 3: friday -= timedelta(days=(weekday + 3)) return friday def is_tracklist_changed( db_tracks: List[SpotifyNewMusicFridayPlaylistTrackHistory], new_tracks: List[SpotifyNewMusicFridayPlaylistTrackHistory], ) -> bool: """Compare two track list if they are different. Args: db_tracks: DB track list. new_tracks: Spotify API track list. Returns: Different or not. """ if len(db_tracks) != len(new_tracks): return True db_tracks = sorted(db_tracks, key=lambda i: i.playlist_index) new_tracks = sorted(new_tracks, key=lambda i: i.playlist_index) for index in range(len(db_tracks)): track_1 = db_tracks[index] track_2 = new_tracks[index] if track_1.track_id != track_2.track_id or track_1.added != track_2.added: return True return False def need_update( db_tracks: List[SpotifyNewMusicFridayPlaylistTrackHistory], new_tracks: List[SpotifyNewMusicFridayPlaylistTrackHistory], friday: date, ) -> bool: """Check if playlist tracks should be updated. Args: db_tracks: DB track list. new_tracks: Spotify API track list. friday: Friday date. Returns: Need update or not. """ if not db_tracks: return True thursday_with_time = datetime.combine(friday, time(0, 0)) - timedelta(days=1) expected_threshold = config.FILTER_THRESHOLD_PERCENTAGE * len(new_tracks) new_tracks_threshold = len([i for i in new_tracks if i.added > thursday_with_time]) if new_tracks_threshold > expected_threshold and is_tracklist_changed(db_tracks, new_tracks): logger.debug("DB: " + ",".join([f"{i.playlist_index}|{i.track_id}|{i.added}" for i in db_tracks])) logger.debug("NEW: " + ",".join([f"{i.playlist_index}|{i.track_id}|{i.added}" for i in new_tracks])) return True return False def print_debug(playlist_ids: List[str], playlist_id: str, index: int, message: str): logger.debug(f"{(index + 1):02d}/{len(playlist_ids)} {playlist_id}: {message}") def update_nmf_tracklist() -> List[str]: """Update NMF playlists tracklists. Returns: List of updated playlist ID. """ updated_playlists = [] try: playlists = main_db.get_playlists() playlist_ids = [i.playlist_id for i in playlists] friday = get_friday() date_id = main_db.get_date(friday) if datetime.utcnow().date().weekday() <= 3: previous_date_id = main_db.get_date(friday - timedelta(days=7)) else: previous_date_id = date_id logger.debug(f"NMF for {friday}. Dates: current {date_id}, previous {previous_date_id}") logger.debug(",".join([f"{i.playlist_id}|{i.last_added_ts}|{i.last_date}" for i in playlists])) for k, playlist_id in enumerate(playlist_ids): playlist_tracks = dsp_api.get_playlist_tracks(playlist_id) new_tracks = [ SpotifyNewMusicFridayPlaylistTrackHistory( date_id=date_id, playlist_id=playlist_id, playlist_index=item["current"], track_id=item["spotify_track"].get("track_id"), added=get_date(item["last_added_date_time"]), isrc=item.get("isrc"), ) for item in playlist_tracks ] db_tracks = main_db.get_playlist_tracks(playlist_id, previous_date_id) if not need_update(db_tracks, new_tracks, friday): print_debug(playlist_ids, playlist_id, k, "up-to-date") continue if not date_id: date_id = main_db.create_date(friday) for track in new_tracks: track.date_id = date_id main_db.update_tracklist(playlist_id, date_id, new_tracks) last_added_ts = max(i.added for i in new_tracks) if new_tracks else None print_debug(playlist_ids, playlist_id, k, f"last date {friday}, last added {last_added_ts}") if last_added_ts: main_db.update_playlist_dates(playlist_id, friday, last_added_ts) updated_playlists.append(playlist_id) except Exception as ex: logger.error(ex) finally: return updated_playlists def calc_and_set_ranks(market_list: List[str]) -> Tuple[List[str], List[str]]: """Update NMF playlists ranks. Args: market_list: Ordered market list. Returns: Known and unknown markets updated. """ if not market_list: return [], [] playlists_market_list = main_db.get_playlists_markets() market_list = [i for i in market_list if i in playlists_market_list] unknown_market_list = list(sorted(i for i in playlists_market_list if i not in market_list)) unknown_from = max(len(market_list) + 1, config.UNKNOWN_MARKET_RANK_FROM) result = { market_code: ( (market_list.index(market_code) + 1) if market_code in market_list else (unknown_market_list.index(market_code) + unknown_from + 1) ) for market_code in playlists_market_list } main_db.set_playlists_ranks(result) return market_list, unknown_market_list def update_nmf_ranks() -> Tuple[List[str], List[str]] or None: """Update NMF playlists ranks. Returns: Ranks updated or not. """ result = None try: last_update = main_db.get_value("nmf_rank_update_ts") if last_update and ( datetime.utcnow() - get_date(last_update)).days < config.UPDATE_RANKS_INTERVAL: return None if config.UPDATE_RANKS_FROM_TODAY: date_to = datetime.utcnow().date() - timedelta(days=3) else: date_to = main_db.get_last_streams_date() date_from = date_to - timedelta(days=28) logger.debug(f"Getting ranks by streams [{date_from}-{date_to}]") markets = main_db.get_markets_order_by_rank(date_from, date_to) logger.debug(f"Markets order {markets}") result = calc_and_set_ranks(markets) main_db.set_value("nmf_rank_update_ts", datetime.utcnow().isoformat()) except Exception as ex: logger.error(ex) finally: return result def main(): """Update NMF tracks and ranks. Job entry point. """ lock = None updated_playlists = [] ranks_market_codes = None try: if config.Redis.LOCK_ENABLED: lock = redis_db.get_redis_lock() if not lock.acquire(blocking=False): logger.warning("Can not get lock") return if config.ENABLE_UPDATE_TRACKLIST: updated_playlists = update_nmf_tracklist() if config.ENABLE_UPDATE_RANKS: ranks_market_codes = update_nmf_ranks() main_db.set_value("nmf_job_run_ts", datetime.utcnow().isoformat()) except Exception as ex: logger.error(ex) raise finally: if config.Redis.LOCK_ENABLED and lock: lock.release() if updated_playlists or ranks_market_codes: eventbridge.put_event(trace_id, updated_playlists, ranks_market_codes) if __name__ == "__main__": main()