from dataclasses import asdict from datetime import timedelta from operator import and_ from typing import List from delphi_api.const import ENABLE_FAKE_DATA from delphi_api.errors import Codes, InvalidInputError from delphi_api.utils import DateUtils, Models from delphi_api.v3.constants import ( ALLOWED_CHART_GROUPS, COUNTRY_WORLDWIDE, DSP_APPLE, DSP_SPOTIFY, TRACK_POS_STREAMS_OFFSET_DAYS, ) from delphi_api.v3.data_models.bigtable.chart_position import ChartPosition from delphi_api.v3.data_models.bigtable.playlist_position import PlaylistPosition from delphi_api.v3.data_models.bigtable.row_key_builder import RowKeyBuilder, RowKeyGroupType from delphi_api.v3.data_models.postgres_db import Chart, Playlist, db from delphi_api.v3.data_models.schemas.models import ( ChartSimpleSchema, ChartTrackPositionSchema, PlaylistSimpleSchema, PlaylistTrackPositionSchema, ) from delphi_api.v3.enums import AggByParam, IncludeParamTracklists, SubsetParam, TracklistType from delphi_api.v3.view_models.fake_utils import FakeUtils from delphi_api.v3.view_models.generic_dsp import GenericDspViewModel from delphi_api.v3.view_models.params import Params from delphi_api.v3.view_models.query_builder import QueryBuilder from delphi_api.v3.view_models.stream import StreamsViewModel class TrackPositionsViewModel: """Track positions data has been consolidated with the streams data as a new column. This static ViewModel is ergo similar to :class:`StreamsViewModel` """ #: These keys values' will be populated in response objects GROUP_BY = ['isrc', 'date', 'dsp'] @staticmethod def _get_fake_track_positions(params: Params): """Temporary method to return data in lieu of BigTable""" return FakeUtils.get_fake_track_positions(params) @classmethod def get_many_charts(cls, params: dict): """Primary method for views to get chart data via models""" params = Params(**params) params.include = params.include or [] params.tracklist_type = TracklistType.CHARTS.value cls.validate_params(params) params = cls._load_additional_params_data(params) result = cls._get_data(params, RowKeyGroupType.CHART_POSITIONS) result = QueryBuilder.sort_results(result, params) result = QueryBuilder.limit_offset_results(result, params) if IncludeParamTracklists.CHARTS.value in params.include: result = cls._load_pg_data_charts(result) return ChartTrackPositionSchema(many=True).dump(result) @classmethod def get_many_playlists(cls, params: dict): """Primary method for views to get playlist data via models""" params = Params(**params) params.include = params.include or [] params.tracklist_type = TracklistType.PLAYLISTS.value cls.validate_params(params) params = cls._load_additional_params_data(params) result = cls._get_data(params, RowKeyGroupType.PLAYLIST_POSITIONS) result = QueryBuilder.sort_results(result, params) result = QueryBuilder.limit_offset_results(result, params) if IncludeParamTracklists.PLAYLISTS.value in params.include: result = cls._load_pg_data_playlists(result) return PlaylistTrackPositionSchema(many=True).dump(result) @staticmethod def validate_params(params: Params): """Check the parameters provided by the client are compatible before continuing""" err = False id_field = 'playlist_id' if params.tracklist_type == TracklistType.PLAYLISTS else 'chart_id' if not (params.artist_id or params.playlist_id or params.chart_id or params.track_id or params.isrc): err = ('Missing at least one identifiying parameter from set: ' f'[artist_id, {id_field}, track_id, isrc]') if params.include: if IncludeParamTracklists.STREAMS.value in params.include and not params.isrc: # this could be removed later, but requests could potentially be very, very large err = 'Missing required parameter `isrc` when providing `include=streams`' if err: raise InvalidInputError({ 'code': Codes.invalid_input.value, 'description': err, }) @staticmethod def _load_additional_params_data(params: Params) -> Params: """ Loads any additional identity data from relational postgres data for use in Bigtable queries Returns: Params: loaded params object with potentially additional IDs from postgres """ params.agg_by = AggByParam.ISRC.value loaded_params = StreamsViewModel.load_additional_params_data(params) # instead of requiring agg_by=isrc, default group all by isrc loaded_params.track_id = None return loaded_params @classmethod def _get_data(cls, params: Params, key_group_type: RowKeyGroupType) -> List[dict]: if ENABLE_FAKE_DATA: return cls._get_fake_track_positions(params) if key_group_type == RowKeyGroupType.PLAYLIST_POSITIONS: return cls._get_playlists_data(params) elif key_group_type == RowKeyGroupType.CHART_POSITIONS: return cls._get_charts_data(params) @classmethod def _get_playlists_data(cls, params: Params) -> List[dict]: """Get the underlying playlist track positions data from data sources""" base_key = 'playlist_id' item_keys = RowKeyBuilder.get_row_keys(params, RowKeyGroupType.PLAYLIST_POSITIONS) model = GenericDspViewModel(params, data_model=PlaylistPosition, base_key_name=base_key) result = model.bigtable_load_results(item_keys).results items = PlaylistTrackPositionSchema(many=True).dump(result) grouped_data = Models.aggregate_generic(items, base_key, group_by=cls.GROUP_BY) result = Models.flatten_generic(grouped_data) if IncludeParamTracklists.STREAMS.value in params.include: # If client provided include=streams, load the streams data return cls._load_streams_data_playlists(params, positions=result) return result @classmethod def _get_charts_data(cls, params: Params) -> List[dict]: """Get the underlying chart track positions data from data sources""" base_key = 'chart_id' item_keys = RowKeyBuilder.get_row_keys(params, RowKeyGroupType.CHART_POSITIONS) model = GenericDspViewModel(params, data_model=ChartPosition, base_key_name=base_key) result = model.bigtable_load_results(item_keys).results items = ChartTrackPositionSchema(many=True).dump(result) grouped_data = Models.aggregate_generic(items, base_key, group_by=cls.GROUP_BY) result = Models.flatten_generic(grouped_data) return result @classmethod def _load_pg_data_playlists(cls, positions: List[dict]) -> List[dict]: """Load Playlist objects data from postgres""" id_key = 'playlist_id' obj_key = 'playlist' ids = QueryBuilder.get_values_from_objs(id_key, positions) q = db.session.query(Playlist).filter(Playlist.playlist_id.in_(ids)) playlists = PlaylistSimpleSchema(many=True).dump(q.all()) return cls._merge_items( parents=positions, children=playlists, id_key=id_key, obj_key=obj_key) @classmethod def _load_pg_data_charts(cls, positions: List[dict]) -> List[dict]: """Load Chart objects data from postgres""" id_key = 'chart_id' obj_key = 'chart' ids = QueryBuilder.get_values_from_objs(id_key, positions) q = db.session.query(Chart).filter( and_(Chart.chart_id.in_(ids), Chart.chart_group.in_(ALLOWED_CHART_GROUPS))) charts = ChartSimpleSchema(many=True).dump(q.all()) return cls._merge_items( parents=positions, children=charts, id_key=id_key, obj_key=obj_key) @classmethod def _load_streams_data_playlists(cls, params: Params, positions: List[dict]) -> List[dict]: """ Notes: We need to request streams regardless of limits on track positions, since the data will not match 1:1. We don't know which playlists the track might not/be in for a single day track position range and n-day range of streams data. Currently the range is fixed at 7 days Args: params: params from track positions request positions: playlist track positions Returns: New list of track positions including streams for fixed range """ id_key = 'playlist_id' obj_key = 'streams' end_date = DateUtils.from_str(params.end_date) start_date = end_date - timedelta(days=TRACK_POS_STREAMS_OFFSET_DAYS) # Create params streams_params = Params( start_date=DateUtils.as_str(start_date), end_date=params.end_date, playlist_id=params.playlist_id, isrc=params.isrc, country_code=[COUNTRY_WORLDWIDE], dsp=params.dsp or [DSP_APPLE, DSP_SPOTIFY], agg_by=AggByParam.ISRC.value, subset=SubsetParam.PLAYLISTS.value, ) streams_params_dict = asdict(streams_params) streams = StreamsViewModel.get_many(params=streams_params_dict) return cls._merge_items(parents=positions, children=streams, id_key=id_key, obj_key=obj_key, obj_field='streams') @classmethod def _merge_items(cls, parents: List[dict], children: List[dict], id_key: str, obj_key: str, obj_field: str = None): """ Args: parents: objects to augment with children children: objects that will be filled into parent keys id_key: link between parents and children obj_key: name of the parent field to update with child object Returns: List[dict]: Copy of parents list updated with children """ id_obj_map = {obj[id_key]: obj for obj in children} items = [] for item in parents: obj = id_obj_map.get(item[id_key], None) if obj: item[obj_key] = obj if not obj_field else obj.get(obj_field) items.append(item) return items