"""Utils and helpers related to streams, skips and saves.""" import datetime from typing import Dict, List from analytics.config import STORES from analytics.constants.parameters import ALL_TIME from analytics.constants.streams import ( STREAMS_SOS_COLUMNS, STREAMS_SOS_COLUMNS_STORE_FILTERS, STREAMS_SOS_DETAILED_COLUMNS, STREAMS_SOS_DETAILED_COLUMNS_STORE_FILTERS, ) from analytics.utils import store_availability STREAMS_BY_COUNTRY_FIELDS = [ "country_code", "date", "streams", "streams_with_skips", "skips", "saves", ] STREAMS_BY_SOS_FIELDS = [ "date", "streams_passive", "streams_active", "streams_collection", "streams", ] STREAMS_BY_SOUND_RECORDING_FIELDS = [ "isrc", "date", "streams", "streams_with_skips", "skips", "saves", ] def _start_date_for_all_time_streams(streams): start_date = min([stream["date"] for stream in streams]) return start_date def calc_date_skip_rate(date_item: dict) -> float: """Calculate Skip Rate for a given date item. Args: date_item (dict): Dict representing streams, skips and saves for a given date. Returns: Skip Rate for the Streams and Skips. """ if not date_item or date_item.get("skips") is None: return None skips = date_item["skips"] streams_with_skips = date_item["streams_with_skips"] return skips / (skips + streams_with_skips) if skips else 0 def breakdown_by_sos(item: Dict, key="value", timeseries=True, sources=None, version=1): """Generate breakdown by source of streams. version is a temporary param, it's going to be removed once we migrate to SoS V2, so it doesn't make sense to make the code DRY. """ sos = [] if version == 1: sources = sources or ["active", "passive", "collection", "unknown"] def _breakdown(item, id): breakdown = { "id": id, key: item.get(f"streams_{id}"), } if not timeseries and "type" in item: breakdown["type"] = item.get("type") if timeseries: breakdown["date"] = item["date"] return breakdown sos = [_breakdown(item, source) for source in sources] elif version == 2: sources = sources or STREAMS_SOS_COLUMNS def _breakdown(item, id): breakdown = {"id": id, key: item.get(f"{id}")} if not timeseries and "type" in item: breakdown["type"] = item.get("type") if timeseries: breakdown["date"] = item["date"] return breakdown sos = [_breakdown(item, source) for source in sources] return sos def get_empty_streams_for_period(start_date, end_date): """Get list of empty streams for given date range. Args: start_date (datetime.date): Start date end_date (datetime.date): End date Returns: List of dictionaries with keys date (str): Date in YYYY-MM-DD format streams (None): Total streams """ start_date = datetime.datetime.strptime(str(start_date), "%Y-%m-%d").date() end_date = datetime.datetime.strptime(str(end_date), "%Y-%m-%d").date() empty_streams = [] for n in range((end_date - start_date).days + 1): empty_streams.append( { "date": (start_date + datetime.timedelta(n)).strftime("%Y-%m-%d"), "streams": 0, "streams_with_skips": 0, "skips": None, "skip_rate": None, "saves": None, } ) return empty_streams def get_streams_for_store(stream_records, store_id, start_date, end_date): """Get streams for store for given date range. Args: stream_records (list): List of dicts representing streams with values store_id (int): Store ID date (datetime.date): Date of streams streams (int): Total streams for date streams_with_skips (int | None): Total streams with skips for date saves (int | None): Total saves for date skips (int | None): Total skips for date store_id (int): Store to get streams for start_date (datetime.date): Start date end_date (datetime.date): End date Returns: List of dictionaries with keys date (str): Date in YYYY-MM-DD format streams (int | None): Total streams for date streams_with_skips (int | None): Total streams with skips for date saves (int | None): Total saves for date skips (int | None): Total skips for date skip_rate (float | None): Total skip rate for date """ streams = get_empty_streams_for_period(start_date, end_date) for record in stream_records: if record["store_id"] == store_id: date_idx = (record["date"] - start_date).days if 0 <= date_idx < len(streams): streams[date_idx]["streams"] = record["streams"] streams[date_idx]["streams_with_skips"] = record["streams_with_skips"] streams[date_idx]["skips"] = record["skips"] streams[date_idx]["saves"] = record["saves"] streams[date_idx]["skip_rate"] = calc_date_skip_rate(record) return streams def get_streams_by_store(streams, start_date, end_date): """Format streams by store for given date range. Args: streams (list): streams result start_date (datetime.date | str): Start date or ALL_TIME keyword end_date (datetime.date): End date Returns: Formatted list of dictionaries with keys id (int): Store ID name (str): Store Name items (list): List of dicts representing streams with values date (str): Date in YYYY-MM-DD format streams (int | None): Total streams for date streams_with_skips (int | None): Total streams with skips saves (int | None): Total saves for date skips (int | None): Total skips for date skip_rate (float | None): Total skip rate for date """ if len(streams) == 0: return streams if start_date == ALL_TIME: start_date = _start_date_for_all_time_streams(streams) store_names = store_availability.get_store_names() all_store_ids = list(set([record["store_id"] for record in streams])) active_store_ids = set(filter(lambda e: e in store_names, all_store_ids)) streams_by_store = [] for store_id in sorted(active_store_ids): streams_by_store.append( { "id": store_id, "name": store_names.get(store_id), "items": get_streams_for_store(streams, store_id, start_date, end_date), } ) return streams_by_store def get_streams_all(stream_records, start_date, end_date): """Get aggregate streams for store for given date range. Args: stream_records (list): List of dicts representing streams with values date (datetime.date): Date of streams streams (int): Total streams for date streams_with_skips (int | None): Total streams with skips for date saves (int | None): Total saves for date skips (int | None): Total skips for date start_date (datetime.date | str): Start date or ALL_TIME keyword end_date (datetime.date): End date Returns: List of dictionaries with keys date (str): Date in YYYY-MM-DD format streams (int | None): Total streams for date saves (int | None): Total saves for date skip_rate (float | None): Total skip rate for date """ if len(stream_records) == 0: return stream_records if start_date == ALL_TIME: start_date = _start_date_for_all_time_streams(stream_records) streams = get_empty_streams_for_period(start_date, end_date) for record in stream_records: date_idx = (record["date"] - start_date).days if 0 <= date_idx < len(streams): streams[date_idx]["streams"] = record["streams"] streams[date_idx]["saves"] = record["saves"] streams[date_idx]["skip_rate"] = calc_date_skip_rate(record) return streams def check_store_ids_for_sos(store_ids: List) -> List: """Validate store ids params against dedicated list of supported store for sos. Args: store_ids (list): List of stores from params Return: store_ids if valid subset of supported stores for sos else error """ store_ids_str = list(map(str, store_ids)) if not set(store_ids_str).issubset(("1", "187", "286", "453")): raise ValueError( "extended SOS only supported for Amazon (187), Apple (1), Spotify (286) and Youtube (453)" ) return store_ids def check_store_ids_for_sos_detailed(store_ids: List) -> List: """Validate store ids for SOS detailed (YouTube not supported). Args: store_ids (list): List of stores from params Return: store_ids if valid subset of supported stores for sos detailed else error """ store_ids_str = list(map(str, store_ids)) if not set(store_ids_str).issubset(("1", "187", "286")): raise ValueError( "SOS detailed only supported for Amazon (187), Apple (1) and Spotify (286)" ) return store_ids def get_streams_sos_columns(store_ids: List[str]) -> List[str]: """Returns a subset of STREAMS_SOS_COLUMNS based on store_ids. If store_ids is empty, returns all STREAMS_SOS_COLUMNS. If the passed store is not supported, an error is raised. """ if not store_ids: return list(STREAMS_SOS_COLUMNS) columns = [] store_ids_str = list(map(str, store_ids)) for store_id in store_ids_str: columns.extend( c for c in STREAMS_SOS_COLUMNS if STREAMS_SOS_COLUMNS_STORE_FILTERS[store_id] in c ) return columns def get_streams_sos_detailed_columns(store_ids: List[str]) -> List[str]: """Returns a subset of STREAMS_SOS_DETAILED_COLUMNS based on store_ids. If store_ids is empty, returns all STREAMS_SOS_DETAILED_COLUMNS. If the passed store is not supported, an error is raised. """ if not store_ids: return list(STREAMS_SOS_DETAILED_COLUMNS) columns = [] store_ids_str = list(map(str, store_ids)) for store_id in store_ids_str: columns.extend( c for c in STREAMS_SOS_DETAILED_COLUMNS if STREAMS_SOS_DETAILED_COLUMNS_STORE_FILTERS[store_id] in c ) return columns def get_streams_sos_columns_from_stream_sources( store_ids: List[str], stream_sources: List[str] ) -> List[str]: """Returns SOS columns filtered by store_ids and stream_sources. Args: store_ids: Store IDs to filter by (e.g. ["1", "286"]). stream_sources: Stream source names to filter by (e.g. ["discovery", "external"]). Returns: List of matching SOS column names. Raises: ValueError: If no valid columns match the combination of store_ids and stream_sources. """ store_names = [ STREAMS_SOS_COLUMNS_STORE_FILTERS[store_id] for store_id in store_ids ] if not stream_sources: return [ col for col in STREAMS_SOS_COLUMNS if any(store_name in col for store_name in store_names) ] sos_columns = [] for store_name in store_names: cols = [ "streams_sos_" + store_name + "_" + source for source in stream_sources if "streams_sos_" + store_name + "_" + source in STREAMS_SOS_COLUMNS ] sos_columns.extend(cols) if not sos_columns: raise ValueError( f"Invalid combination of stream sources: {stream_sources} for store_ids: {store_ids}" ) return sorted(set(sos_columns)) def get_streams_sos_detailed_columns_from_stream_sources( store_ids: List[str], stream_sources: List[str] ) -> List[str]: """Returns SOS detailed columns filtered by store_ids and stream_sources. Args: store_ids: Store IDs to filter by (e.g. ["1", "286"]). stream_sources: Stream source names to filter by (e.g. ["discovery", "external"]). Returns: List of matching SOS detailed column names. Raises: ValueError: If no valid columns match the combination of store_ids and stream_sources. """ store_names = [ STREAMS_SOS_DETAILED_COLUMNS_STORE_FILTERS[store_id] for store_id in store_ids ] if not stream_sources: return [ col for col in STREAMS_SOS_DETAILED_COLUMNS if any(store_name in col for store_name in store_names) ] sos_columns = [] for store_name in store_names: cols = [ "streams_sos_" + store_name + "_" + source for source in stream_sources if "streams_sos_" + store_name + "_" + source in STREAMS_SOS_DETAILED_COLUMNS ] sos_columns.extend(cols) if not sos_columns: raise ValueError( f"Invalid combination of stream sources: {stream_sources} for store_ids: {store_ids}" ) return sorted(set(sos_columns)) def clean_up_dates_from_lag( timeseries_to_update, timeseries_key, dates_with_no_data_to_add, query_params, missing_dates_by_store, ): """Add data point in timeseries for missing dates with has_missing_data property to indicates whether this is because of ingestion lag or not. Args: timeseries_to_update (list): List of dict representing timeseries data timeseries_key (str): Optional String representing the key for timeseries data dates_with_no_data_to_add (set): Set of dates representing gap from requested range query_params (dict): Dict representing params from endpoint missing_dates_by_store (list): List of dict representing missing dates based on store id Return: updated timeseries data """ updated_timeseries = timeseries_to_update.copy() found_missing_dates = [] if ( query_params["dimension"] == "SOS" or query_params["dimension"] == "SOS_V2" or query_params["dimension"] == "COUNTRY" ) and len(query_params["store_ids"]) == 1: found_missing_dates = [ item["date"] for item in missing_dates_by_store if item["id"] == query_params["store_ids"][0] ] elif query_params["dimension"] == "STORE": if timeseries_key: store_id_key = [k for k, v in STORES.items() if v == timeseries_key][0] found_missing_dates = [ item["date"] for item in missing_dates_by_store if item["id"] == store_id_key ] else: if len(query_params["store_ids"]) == 1: found_missing_dates = [ item["date"] for item in missing_dates_by_store if item["id"] == query_params["store_ids"][0] ] for date in dates_with_no_data_to_add: missing_date = {"date": date, "value": 0, "has_missing_data": False} if date in found_missing_dates: missing_date["has_missing_data"] = True updated_timeseries.append(missing_date) else: updated_timeseries.append(missing_date) return updated_timeseries