import copy from datetime import datetime, timedelta from typing import Dict, Iterator, List from playlist.queries.constants import ( ADD_CURRENT_EVENT, ADD_HISTORY_EVENT, CHANGE_EVENT, CONSECUTIVE_EVENTS_TIMESTAMP_THRESHOLD, REMOVAL_EVENT, ) def _daterange(start_date: datetime, end_date: datetime) -> Iterator[str]: """Generate date string in the range [start_date, end_date).""" for n in range(int((end_date - start_date).days)): date = start_date + timedelta(n) yield date.strftime("%Y-%m-%d") def _date_to_str(date: datetime) -> str: """Convert date to string.""" return date.strftime("%Y-%m-%d") def _str_to_date(date_str: str) -> datetime: """Convert a string to date.""" return datetime.strptime(date_str, "%Y-%m-%d") def _str_to_timestamp(timestamp_str: str) -> datetime: """Convert a string to timestamp.""" return datetime.strptime(timestamp_str, "%Y-%m-%dT%H:%M:%S.%fZ") def _timestamp_to_str(timestamp: datetime) -> str: """Convert a timestamp to a string.""" return timestamp.strftime("%Y-%m-%dT%H:%M:%S.%fZ") def _abs_timestamp_difference(timestamp1: str, timestamp2: str) -> float: """Absolute number of seconds between two timestamps.""" ts1 = _str_to_timestamp(timestamp1) ts2 = _str_to_timestamp(timestamp2) return abs((ts1 - ts2).total_seconds()) def _get_last_position_event(sublist: List[Dict]) -> Dict: """Get the event with the latest timestamp that has a non-null position. Falls back to the last element if every position is None. """ best = None for item in sublist: if item["position"] is not None: if best is None or item["timestamp"] >= best["timestamp"]: best = item return best if best is not None else sublist[-1] def _get_missing_position_data(prev_position_data_item: Dict) -> Dict: """Construct the missing position data from previous item.""" new_item = copy.deepcopy(prev_position_data_item) # Increment the previous timestamp by 1 day new_item_timestamp = _str_to_timestamp(new_item["timestamp"]) + timedelta(days=1) new_item["timestamp"] = _timestamp_to_str(new_item_timestamp) new_item["date"] = _date_to_str(new_item_timestamp) return new_item def _format_timeseries_item(item: Dict) -> Dict: """Formats timeseries item.""" item.pop("timestamp", None) item["timestamp"] = _timestamp_to_str(_str_to_date(item.pop("date", None))) item.pop("event_priority", None) item.pop("event_type", None) return item def fill_in_missing_position_values(position_time_series: List[dict]) -> List[dict]: """ Fill in missing position data in time series based on event event_type. Args: position_time_series: The time series to be completed Returns: The complete time series """ complete_position_time_series = [] i = 0 while i < len(position_time_series): if ( i + 1 < len(position_time_series) and position_time_series[i]["event_type"] in [ADD_HISTORY_EVENT, ADD_CURRENT_EVENT] and position_time_series[i]["position"] is None and position_time_series[i + 1]["position"] is not None ): time_series_item = position_time_series[i].copy() time_series_item["position"] = position_time_series[i + 1]["position"] complete_position_time_series.append(time_series_item) else: complete_position_time_series.append(position_time_series[i]) i = i + 1 return complete_position_time_series def filter_change_after_removal_events(position_time_series: List[dict]) -> List[dict]: """ Filter removal events after change if occur within same timestamps. This filtering is required as there is a redundant change event recorded after every removal event. Args: position_time_series: The time series to be filtered Returns: The filtered time series """ filtered_position_time_series = [] i = 0 while i < len(position_time_series): if ( i + 1 < len(position_time_series) and position_time_series[i]["event_type"] == REMOVAL_EVENT and position_time_series[i + 1]["event_type"] == CHANGE_EVENT and _abs_timestamp_difference( position_time_series[i]["timestamp"], position_time_series[i + 1]["timestamp"], ) <= CONSECUTIVE_EVENTS_TIMESTAMP_THRESHOLD ): # Keep the CHANGE event but preserve the REMOVAL event's removed_on date. # This is correct because filter_add_and_removal_events runs first and # filters out ADD+REMOVAL pairs (Spotify ID changes). Any REMOVAL+CHANGE # pair that reaches here is a real removal, so we use the REMOVAL's removed_on. merged_event = position_time_series[i + 1].copy() merged_event["removed_on"] = position_time_series[i]["removed_on"] filtered_position_time_series.append(merged_event) i = i + 2 else: filtered_position_time_series.append(position_time_series[i]) i = i + 1 return filtered_position_time_series def filter_add_and_removal_events(position_time_series: List[dict]) -> List[dict]: """ Filter consecutive add and removal events if occur within same timestamps. This indicates that the track with the isrc was not removed but just the spotify id changed. Args: position_time_series: The time series to be filtered Returns: The filtered time series """ filtered_position_time_series = [] i = 0 while i < len(position_time_series): if ( i + 1 < len(position_time_series) and position_time_series[i]["event_type"] in [ADD_HISTORY_EVENT, ADD_CURRENT_EVENT] and position_time_series[i + 1]["event_type"] == REMOVAL_EVENT and _abs_timestamp_difference( position_time_series[i]["timestamp"], position_time_series[i + 1]["timestamp"], ) <= CONSECUTIVE_EVENTS_TIMESTAMP_THRESHOLD ): i = i + 2 else: filtered_position_time_series.append(position_time_series[i]) i = i + 1 return filtered_position_time_series def interpolate_missing_position_time_series_values( position_time_series: List[dict], start_date_str: str = None, end_date_str: str = None, ) -> List[dict]: """Interpolate missing position time series values. Position time series data is not recorded for every time interval which leaves gaps in the timeseries. This function will fill in the gaps for dates for which data is not present. Given a time series it will extract the start date and construct a complete timeseries till current date. After which it will filter the timeseries for dates in the range (start_date_str, end_date_str). This function guarantees that there is at least one entry for each date. Args: position_time_series: The time series to be filtered start_date_str: Starting date of position time series in string end_date_str: Ending date of position time series in string Returns: A complete representation of time series data to current date. """ if len(position_time_series) <= 0: return [] min_date = _str_to_date(position_time_series[0]["date"]) max_date = datetime.now() # Generate dates in the range (min_date, max_date) and initialize the # dictionary as buckets of dates to their corresponding position data date_to_time_series_item = { date: [] for date in _daterange(min_date, max_date + timedelta(days=1)) } for item in position_time_series: # Iterate through the position timeseries and append the item in the # corresponding bucket for the date date_to_time_series_item[item["date"]].append(item) for curr_date_str, items in date_to_time_series_item.items(): # Iterate through all the keys to determine if any date is missing # position data and fill it in appropriately curr_date = _str_to_date(curr_date_str) if len(items) == 0: # No position change was recorded for this date # Copy the latest position from last item and modify the date prev_date = curr_date - timedelta(days=1) prev_date_str = _date_to_str(prev_date) missing_position_data = _get_missing_position_data( date_to_time_series_item[prev_date_str][-1] ) date_to_time_series_item[curr_date_str].append(missing_position_data) # Filtering results according to start and end date start_date = _str_to_date(start_date_str) if start_date_str else min_date end_date = _str_to_date(end_date_str) if end_date_str else max_date if start_date < min_date: start_date = min_date if end_date > max_date: end_date = max_date complete_time_series = [] # Collect the time series from all buckets for date_key in _daterange(start_date, end_date + timedelta(days=1)): sublist = date_to_time_series_item[date_key] item = _get_last_position_event(sublist) formatted_item = _format_timeseries_item(item) complete_time_series.append(formatted_item) return complete_time_series def remove_positions_after_removed_on_time_series_values( position_time_series: List[dict], ) -> List[dict]: """Remove positions after removed_on date. Args: position_time_series: The time series to be transformed. Returns: A complete representation of time series data to current date. """ if not position_time_series: return [] for item in position_time_series: # Iterate through the position timeseries and remove position if # a track was removed from a playlist if item["timestamp"] >= item["removed_on"]: item["position"] = None del item["removed_on"] return position_time_series def transform_time_series( position_time_series: List[dict], start_date_str: str = None, end_date_str: str = None, ) -> List: # fmt: off # Note: filter_add_and_removal_events must run BEFORE filter_change_after_removal_events # because we need to detect ADD+REMOVAL pairs (indicating a Spotify ID change, not a real # removal) before the REMOVAL+CHANGE merger obscures the REMOVAL event. position_time_series = filter_add_and_removal_events(position_time_series) position_time_series = filter_change_after_removal_events(position_time_series) position_time_series = fill_in_missing_position_values(position_time_series) position_time_series = interpolate_missing_position_time_series_values( position_time_series, start_date_str, end_date_str ) position_time_series = remove_positions_after_removed_on_time_series_values( position_time_series ) # fmt: on return position_time_series