"""Update existing playlist streams/followers data.""" from datetime import date, timedelta from typing import Dict, List, Tuple from apollo_main_db.spotify.models import SpotifyPlaylistFollowersWeekHistory, SpotifyPlaylistStreamsWeekHistory import config import main_db import utils from constants.common import WORLDWIDE, Table from logger import logger from metric_data import MetricData, get_metric_data def set_values( record: SpotifyPlaylistStreamsWeekHistory, country_code: str, values_dict: Dict[str, int] ): """Set object values. Args: record: Object. country_code: Country code. values_dict: Current/previous values. """ setattr(record, "value" if country_code == WORLDWIDE else f"value_{country_code}", values_dict["current"]) setattr( record, "change" if country_code == WORLDWIDE else f"change_{country_code}", values_dict["current"] - values_dict["previous"], ) def process_streams( data: Dict[str, Dict[str, Dict[str, int]]], current_date: date, day_7_date: date, day_14_date: date ) -> Tuple[List[SpotifyPlaylistStreamsWeekHistory], List[str]]: """Process day-by-day streams data. Args: data: Playlist ID to country code to current/previous streams mapping. current_date: Current date (second/latest week end). day_7_date: Current - 7 days date. day_14_date: Current - 14 days date. Returns: DB week streams history records and updated playlist ID list. """ data_records = [] for playlist_id, country_code_data in data.items(): record = SpotifyPlaylistStreamsWeekHistory() record.playlist_id = playlist_id record.date = current_date data_records.append(record) for country_code, streams_data in country_code_data.items(): set_values(record, country_code, streams_data) return data_records, list(data.keys()) def process_followers( data: list, current_date: date, day_7_date: date, day_14_date: date ) -> Tuple[List[SpotifyPlaylistFollowersWeekHistory], List[str]]: """Process followers day-by-day data into records with two sequential weeks followers sums. Args: data: Raw day-by-day followers data. current_date: Current date (second/latest week end). day_7_date: Current - 7 days date. day_14_date: Current - 14 days date. Returns: DB week followers history records and updated playlist ID list. """ data_records, updated_playlists = {}, [] for row in data: if row.playlist_id in data_records: record = data_records[row.playlist_id] else: record = SpotifyPlaylistFollowersWeekHistory() record.date = current_date record.playlist_id = row.playlist_id record.value = 0 record.change = 0 data_records[row.playlist_id] = record # value = current date total followers - %current-7days% date total followers # change = current date total followers + %current-14days% total followers # - 2 * %current-7days% date total followers if row.date == current_date: updated_playlists.append(record.playlist_id) record.value += row.followers record.change += row.followers elif row.date == day_7_date: record.value -= row.followers record.change -= row.followers * 2 elif row.date == day_14_date: record.change += row.followers # add aggregated rows only if current date followers records exist return [v for k, v in data_records.items() if k in updated_playlists], updated_playlists @utils.timing def update_one_date(metric_data: MetricData, current_date: date): """Update data for single date. Args: metric_data: Metric data. current_date: Current date to process. """ table_type = metric_data.table_type aggregated_playlists = metric_data.db.get_aggregated_playlists(current_date) if not aggregated_playlists: logger.debug(f"{table_type}: {current_date} up-to-date") return day_7_date = current_date - timedelta(days=7) day_14_date = current_date - timedelta(days=14) logger.debug(f"{table_type}: {current_date} {len(aggregated_playlists)}") process_data = process_streams if table_type == Table.STREAMS else process_followers chunk_size = config.PLAYLIST_UPDATE_METRIC_CHUNK_SIZE[table_type] items_count = len(aggregated_playlists) throttle_logger = utils.ThrottleLogger(config.LOG_THROTTLE_UPDATE_CHUNK_COUNT[table_type], chunk_size, items_count) for index in range(0, items_count, chunk_size): playlist_chunk = aggregated_playlists[index : index + chunk_size] data = metric_data.get_data(playlist_chunk, current_date, day_7_date, day_14_date) if not data: continue data_records, updated_playlists = process_data(data, current_date, day_7_date, day_14_date) throttle_logger.debug(f"{table_type}: {current_date} {index}/{items_count}", index, len(data_records)) if not data_records: continue metric_data.db.save_data(data_records) metric_data.db.set_latest_dates( [ {"playlist_id": playlist_id, "latest_date": current_date} for playlist_id in updated_playlists ] ) def update_data(update_table_list: List[str]): """Update streams and followers. Args: update_table_list: What tables types to update. """ for table_type in update_table_list: metric_data = get_metric_data(table_type) for index in range((metric_data.latest_date - metric_data.previous_date).days + 1): current_date = metric_data.previous_date + timedelta(days=index) update_one_date(metric_data, current_date) @utils.timing def update_metric_playlists(table_type: str): """Update changed playlists. Args: table_type: Table type. """ db = main_db.db[table_type] chunk_size = config.PLAYLIST_UPDATE_METRIC_CHUNK_SIZE[table_type] changed_playlists = db.get_metric_changed_playlists() for index in range(0, len(changed_playlists), chunk_size): playlist_chunk = changed_playlists[index: index + chunk_size] count = db.update_metric_playlists(playlist_chunk) logger.debug(f"{table_type}: {index}/{len(changed_playlists)} {count} updated")