import json from dataclasses import asdict, dataclass, field from datetime import date, datetime, timedelta from json import JSONDecodeError from typing import Any, Dict, List, Tuple, Union from client import service from config import TRACK_MIN_EVENT_MOVE from constants.common import (BREAKDOWN, CHART_TYPE, DELPHI_CHART_UPDATED_KEY, DELPHI_CHARTS_RESULT_FIELD, DSP, ChartStatus) from logger import logger from utils import check_artist, remove_milliseconds def get_chart_data_health_status(dsp: DSP) -> Dict[str, Any]: """Get full information about chart updated statuses from Delphi Charts Data-Health Api for specific DSP such as apple or spotify We pass status=["complete", "complete_volatile"] to get information only for updates completed charts Args: dsp: DSP (ex: DSP.SPOTIFY, DSP.APPLE etc) Returns: Dict Data-Health charts status information """ return service.dsp.get_charts_data_health_status( dsp=dsp, status=[ChartStatus.COMPLETE.value, ChartStatus.COMPLETE_VOLATILE.value] ) def get_charts_last_update_as_dict(dsp: DSP, data_health_data: Dict) -> Dict[str, str]: """Get chart_id to chart max_date (Delphi last updated date) status information Dict. We create delphi sampled chart_id(s) that consists from __ as dict keys data_to_process = key to get charts that we are interested in (ex: breakdown_chart_type_country_code) Args: dsp: DSP (ex: DSP.SPOTIFY, DSP.APPLE etc) data_health_data: Raw Dict data got from Delphi Charts Data-Health Api Returns: Dict Data-Health charts status information """ data_to_process = DELPHI_CHARTS_RESULT_FIELD[dsp] breakdown = BREAKDOWN[dsp] chart_type = CHART_TYPE[dsp] chart_prefix = f"{chart_type}_{breakdown}_" if dsp == DSP.SPOTIFY else f"{chart_type}_" data = data_health_data[data_to_process][breakdown][chart_type] data = {f"{chart_prefix}{key}": value[DELPHI_CHART_UPDATED_KEY[dsp]] for key, value in data.items()} return data def get_charts_ids_to_process( delphi_data: Dict[str, str], apollo_data: Dict[str, Dict[str, str]] ) -> Dict[str, Dict[str, str]]: """Compare last chart updated information from DB to Delphi and get updated charts that need to be processed by job Args: delphi_data: Dict created from Delphi Data-Health Status data apollo_data: Dict created from tblChartsMonitoring data Returns: Dict of charts_ids to dict of dates information """ result = dict() for chart_id, latest_update_date in delphi_data.items(): latest_update_date = remove_milliseconds(latest_update_date) chart_last_updated_datetime = apollo_data.get(chart_id, {}).get("chart_datetime") if not chart_last_updated_datetime or datetime.fromisoformat(latest_update_date) > datetime.fromisoformat( chart_last_updated_datetime ): logger.info( f"{chart_id} - chart is updated at Delphi:" f" \n Previous update {chart_last_updated_datetime},\n" f"Last Delphi update {latest_update_date}" ) result[chart_id] = { "current_updated": latest_update_date, "previous_updated": chart_last_updated_datetime, "data": apollo_data.get(chart_id, {}).get("data", None), } return result def jsonify_string(string: str) -> Dict[str, Any]: try: result = json.loads(string) except JSONDecodeError: result = json.loads(string.replace("'", '"')) return result def get_current_previous_chart( dsp: DSP, chart_id: str, current_date: str, previous_chart: Union[str, None] ) -> Tuple[Dict[str, Any], Dict[str, Any]]: """Get current and previous chart tracks from Delphi Charts Tracks Api and convert it to track_id to full track information Args: dsp: DSP (ex: DSP.SPOTIFY, DSP.APPLE etc) chart_id: Delphi sample chart_id (ex: spotify - regional_daily_us, apple - charts_daily_us) current_date: actual last updated "max_date" from Delphi charts data health api previous_chart: previous chart state got from apollo api Returns: Tuple of current and previous days charts tracks information """ current_charts_data = service.dsp.get_charts_tracks( dsp=dsp, chart_id=chart_id, metrics_dimension="track_id", date=current_date ) current_charts_dict = get_delphi_charts_data_as_dict(current_charts_data) if previous_chart and dsp == DSP.APPLE: return current_charts_dict, jsonify_string(previous_chart) previous_date = (date.fromisoformat(current_date) - timedelta(days=1)).isoformat() previous_charts_data = service.dsp.get_charts_tracks( dsp=dsp, chart_id=chart_id, metrics_dimension="track_id", date=previous_date ) previous_charts_dict = get_delphi_charts_data_as_dict(previous_charts_data) return current_charts_dict, previous_charts_dict def get_delphi_charts_data_as_dict(data: Dict) -> Dict[str, Any]: """Create chart_id to full data dict Args: data: Raw Delphi Charts Tracks Api data Returns: Dict """ return {item["public_meta"]["track_id"]: item for item in data["items"]} def get_chart_events( current_chart: Dict[str, Any], previous_chart: Dict[str, Any] ) -> Tuple[List[Dict[str, Any]], List[Dict[str, Any]], List[Dict[str, Any]], datetime, List[Dict[str, Any]]]: """Get events such as additions/removals/moves Args: current_chart: dict of current chart track_id to full track data previous_chart: dict of previous chart track_id to full track data Returns: List of additions, List of removals, List of moves, datetime when chart was processed """ broken_data = [] additions = [] moves = [] for track_id, data in current_chart.items(): if not check_artist(data): broken_data.append(data) if track_id in previous_chart.keys(): previous_chart.pop(track_id) continue previous_data = previous_chart.get(track_id) if not previous_data: additions.append(get_event(data)) continue if abs(data["metrics"]["position"] - previous_data["metrics"]["position"]) >= TRACK_MIN_EVENT_MOVE: moves.append(get_event(data, previous_data)) previous_chart.pop(track_id) removals = get_removals(previous_chart.values()) processed_datetime = datetime.utcnow() return additions, removals, moves, processed_datetime, broken_data @dataclass class EventTrack: """Nested track information for any event""" id: str isrc: str name: str image_url: str artists: list = field(default_factory=list) def __post_init__(self): self.check_artists() self.change_artists_keys() def check_artists(self): if not self.artists: self.artists = [] return self.artists def change_artists_keys(self): for item in self.artists: artist_id = item.pop("artist_id", None) if artist_id: item["id"] = artist_id return self.artists @dataclass class Event: """Dataclass to create events such as additions/removals/moves""" track: EventTrack current_position: (int, None) = None previous_position: (int, None) = None def get_track_positions( current_track: Dict[str, Any], previous_track: Dict[str, Any] = None, removal: bool = False ) -> Tuple[Union[int, None], Union[int, None]]: """Get current and previous track positions in a chart Args: current_track: current day track in particular chart information previous_track: previous ^^ --""--""-- removal: switch to point that track was removed from chart Returns: Tuple of (current, past) positions of track in chart """ last_available_position = current_track["metrics"]["position"] if not removal: current_position = last_available_position previous_position = previous_track["metrics"]["position"] if previous_track else None else: current_position = None previous_position = last_available_position return current_position, previous_position def get_event( current_track: Dict[str, Any], previous_track: Dict[str, Any] = None, removal: bool = False ) -> Dict[str, Any]: """Get event such as addition/removal/move Args: current_track: current day track in particular chart information previous_track: previous ^^ --""--""-- removal: switch to point that track was removed from chart Returns: event dict of a specific format """ current_position, previous_position = get_track_positions(current_track, previous_track, removal) event = Event( track=EventTrack( id=current_track["public_meta"]["track_id"], isrc=current_track["public_meta"]["isrc"], name=current_track["public_meta"]["name"], image_url=current_track["public_meta"]["image_url"], artists=current_track["public_meta"]["artists"], ), current_position=current_position, previous_position=previous_position, ) return asdict(event) def get_removals(tracks) -> List[Dict[str, Any]]: """Get list of removals events Args: tracks: Dict _dict_values of tracks that were removed from chart Returns: List of removal events """ return [get_event(track, removal=True) for track in tracks]