from collections import defaultdict from datetime import date, datetime, timedelta from typing import Any, Dict, Iterable, List, Optional, Tuple, Union from apollo_main_db.spotify.models import SpotifyHotHitsPlaylistTrackStats import config import constants.common as consts import dsp_api import main_db import utils from logger import logger def get_date(date_str: Optional[str], with_time: bool = False) -> Optional[Union[date, datetime]]: if not date_str: return None if "." in date_str and len(date_str) < 26: date_str += "0" * (26 - len(date_str)) result = datetime.fromisoformat(date_str) return result if with_time else result.date() def get_tracks_peak(playlist_id: str, isrc_list: Iterable[str], current_date: date) -> Dict[str, dict]: position_list = dsp_api.get_playlists_tracks([playlist_id], isrc_list, consts.MIN_DATE, current_date) result = {} for item in position_list: isrc, position = item.get("isrc"), item.get("current") if not isrc or position is None: continue if isrc not in result or result[isrc]["peak_position"] > position: result[isrc] = {"peak_date": get_date(item.get("date")), "peak_position": position} return result def get_playlists_tracks( playlist_id_list: Iterable[str], isrc_list: Iterable[str] = None, current_date: Optional[date] = None, with_dates: bool = False, ) -> Union[Tuple[Dict[str, Dict[str, Any]], Dict[str, date], datetime], Dict[str, Dict[str, Any]]]: track_list = dsp_api.get_playlists_tracks( playlist_id_list, isrc_list, start_date=current_date, end_date=current_date ) stats_map = defaultdict(dict) max_ts = None max_date_map = {} for item in track_list: playlist_id, isrc, latest_ts = ( item.get("playlist_id"), item.get("isrc"), get_date(item.get("latest_position_date_time"), with_time=True) ) if not playlist_id or not isrc or not item.get("is_top_track_for_isrc") or not latest_ts: continue playlist_id, latest_date = playlist_id.replace("spotify_", ""), latest_ts.date() if with_dates: if not max_ts or (latest_ts and latest_ts > max_ts): max_ts = latest_ts if playlist_id not in max_date_map or latest_date and latest_date > max_date_map[playlist_id]: max_date_map[playlist_id] = latest_date stats_map[playlist_id][isrc] = { "playlist_id": playlist_id, "isrc": isrc, "entry_date": ( get_date(item.get("earliest_position_date_time") or item.get("last_added_date_time")) or latest_date ), "latest_date": latest_date, "latest_position": item.get("current"), "previous_date": item.get("last_date_change_14_days"), "previous_position": item.get("previous_position_change_14_days"), "peak_date": latest_date, "peak_position": item.get("current"), "added_date": get_date(item.get("last_added_date_time")), "in_history": True, } return (stats_map, max_date_map, max_ts) if with_dates else stats_map def is_different(new_item: dict, old_item: SpotifyHotHitsPlaylistTrackStats) -> bool: for field in consts.FIELD_LIST: if new_item[field] is not None and getattr(old_item, field) != new_item[field]: return True def update_one_date(playlist_id_list: List[str], current_date: date, stats_map: Dict[str, Dict[str, Dict[str, Any]]]): total_metrics = {consts.Command.DELETE: 0, consts.Command.UPDATE: 0, consts.Command.INSERT: 0} if not stats_map: stats_map = get_playlists_tracks(playlist_id_list, current_date=current_date) for playlist_id in playlist_id_list: current_metrics = {} current_map = stats_map.get(playlist_id, {}) isrc_list = set(current_map.keys()) # delete items that not in current isrc list and have entry date == current current_metrics[consts.Command.DELETE] = main_db.stats.delete_extra(playlist_id, isrc_list, current_date) old_stats_map = {i.isrc: i for i in main_db.stats.get(playlist_id, isrc_list, current_date)} old_isrc_list = set(old_stats_map.keys()) # get items that not in stats table and add them add_isrc_list = isrc_list - old_isrc_list if add_isrc_list: main_db.stats.add(playlist_id, [current_map[i] for i in add_isrc_list]) current_metrics[consts.Command.INSERT] = len(add_isrc_list) # get items that not in tracklist but were in stats before current date, calc stats for them and update rollback_isrc_list = old_isrc_list - isrc_list if rollback_isrc_list: rollback_stats_map = get_playlists_tracks(playlist_id_list=[playlist_id], isrc_list=rollback_isrc_list) rollback_stats_map = rollback_stats_map.get(playlist_id, {}) peak_isrc_list = [ isrc for isrc, data in rollback_stats_map.items() if data["peak_date"] == data["latest_date"] ] if peak_isrc_list: logger.debug(f"{playlist_id} roll back peak positions {peak_isrc_list}") rollback_peak_map = get_tracks_peak(playlist_id, peak_isrc_list, current_date) for isrc, peak_data in rollback_peak_map.items(): rollback_stats_map[isrc].update(peak_data) if rollback_stats_map: main_db.stats.update(playlist_id, rollback_stats_map) current_metrics[consts.Command.UPDATE] = len(rollback_isrc_list) not_found_isrc_list = rollback_isrc_list - set(rollback_stats_map.keys()) if not_found_isrc_list: current_metrics[consts.Command.DELETE] += main_db.stats.delete(playlist_id, not_found_isrc_list) # potential update list, compare DB and Delphi API records, set changed possible_upd_isrc_list = old_isrc_list & isrc_list upd_stats_map = {} for isrc in possible_upd_isrc_list: old_item = old_stats_map[isrc] new_item = current_map[isrc] if new_item["peak_position"] is None or new_item["peak_position"] >= old_item.peak_position: new_item["peak_position"] = old_item.peak_position new_item["peak_date"] = old_item.peak_date if is_different(new_item, old_item): upd_stats_map[isrc] = new_item if upd_stats_map: main_db.stats.update(playlist_id, upd_stats_map) current_metrics[consts.Command.UPDATE] = current_metrics.get(consts.Command.UPDATE, 0) + len(upd_stats_map) if current_metrics: logger.debug(f"{playlist_id} - {current_metrics}") else: logger.debug(f"{playlist_id} is up-to-date") for command in consts.Command.ALL_MODIFY: total_metrics[command] += current_metrics.get(command, 0) logger.info(f"{current_date} {total_metrics}") @utils.timing def update_stats() -> Tuple[Dict[str, date], date, datetime]: """Update stats data.""" previous_playlist_date_map = {i.playlist_id: i.last_date for i in main_db.get_hot_hits_playlists()} playlist_id_list = list(previous_playlist_date_map.keys()) date_list = [i for i in previous_playlist_date_map.values() if i] previous_date = max(date_list) if date_list else None stats_map, current_playlist_date_map, max_ts = get_playlists_tracks(playlist_id_list, with_dates=True) hot_hits_date = max(current_playlist_date_map.values()) # update from backfill date or from the previous hot hits date from_date = min(config.DATE_FROM, previous_date or config.DATE_FROM) # update up to the current hot hits date to_date = min(hot_hits_date, config.DATE_TO) logger.debug(f"stats: {from_date}-{to_date} updating") current_date = from_date while current_date <= to_date: update_one_date(playlist_id_list, current_date, stats_map if to_date == current_date else None) current_date += timedelta(days=1) return current_playlist_date_map, hot_hits_date, max_ts def calc_and_set_ranks(market_list: List[str]) -> Tuple[List[str], List[str]]: """Update HH playlists ranks. Args: market_list: Ordered market list. Returns: Known and unknown markets. """ if not market_list: return [], [] # AP-8045: global should have rank = 1 if consts.MARKET_GLOBAL not in market_list: market_list.insert(0, consts.MARKET_GLOBAL) playlists_market_list = [i.market_code.lower() for i in main_db.get_playlists_markets()] known_market_list = [i for i in market_list if i in playlists_market_list] unknown_market_list = list(sorted(i for i in playlists_market_list if i not in known_market_list)) unknown_from = max(len(known_market_list) + 1, config.UNKNOWN_MARKET_RANK_FROM) result = { market_code: ( (known_market_list.index(market_code) + 1) if market_code in known_market_list else (unknown_market_list.index(market_code) + unknown_from + 1) ) for market_code in playlists_market_list } main_db.set_playlists_ranks(result) return known_market_list, unknown_market_list def update_ranks() -> Tuple[List[str], List[str]] or None: """Update HH playlists ranks. Returns: Ranks updated or not. """ result = None try: last_update = main_db.get_value(consts.ApolloKey.HH_RANK_UPDATE) if last_update and ( datetime.utcnow() - datetime.fromisoformat(last_update)).days < config.UPDATE_RANKS_INTERVAL: return None if config.UPDATE_RANKS_FROM_TODAY: date_to = datetime.utcnow().date() - timedelta(days=3) else: date_to = main_db.get_last_streams_date() date_from = date_to - timedelta(days=28) logger.debug(f"Getting ranks by streams [{date_from}-{date_to}]") markets = main_db.get_markets_order_by_rank(date_from, date_to) logger.debug(f"Markets order {markets}") result = calc_and_set_ranks(markets) main_db.set_value(consts.ApolloKey.HH_RANK_UPDATE, datetime.utcnow().isoformat()) except Exception as ex: logger.error("Ranks update error: ", ex) finally: return result