from datetime import date, datetime, time, timedelta from typing import Dict, List import pytest from apollo_main_db.apollo.models import ApolloKeyValueStorage, MarketRegionTypeEnum from apollo_main_db.spotify.models import SpotifyHotHitsPlaylistTrackStats, SpotifyMarketRank, ViewPlaylistTypeEnum from sqlalchemy.orm import Session import config import index as index_module import update from constants import common as consts from tests import factories def get_ts( year: int = 2021, month: int = 6, day: int = 14, hour: int = 15, minute: int = 34, second: int = 51 ) -> datetime: return datetime(year, month, day, hour, minute, second) def get_db_value(db_session: Session, key: str) -> str: return ( db_session.query(ApolloKeyValueStorage.value).filter(ApolloKeyValueStorage.key == key).first() )[0] @pytest.mark.parametrize( "updated_ts,current_ts,update_interval,expected_ts,expected_ranks", ( (get_ts(day=14), get_ts(day=16), 4, get_ts(day=14), {"es": 1, "mArKeT1": 2, "US": 3, "Market2": 4, "gb": 5}), (get_ts(day=14), get_ts(day=16), 1, get_ts(day=16), {"US": 1, "gb": 2, "es": 3, "mArKeT1": 51, "Market2": 52}), ), ) def test_update_hh_ranks(updated_ts, current_ts, update_interval, expected_ts, expected_ranks, mocker, db_session): streams_markets = ["us", "gb", "ca", "es"] playlists_markets = ["es", "mArKeT1", "US", "Market2", "gb"] mocked_datetime = mocker.patch.object(update, "datetime", mocker.Mock(wraps=datetime)) mocked_datetime.utcnow.return_value = current_ts config.UPDATE_RANKS_INTERVAL = update_interval config.UNKNOWN_MARKET_RANK_FROM = 50 factories.ApolloKeyValueStorageFactory.create(key="hh_rank_update_ts", value=updated_ts.isoformat()) for i, market in enumerate(playlists_markets): factories.ApolloMarketFactory.create(id=(i + 1), code=market) factories.ApolloMarketFactory.create(id=(i + 101), code=f"mm_{i}") factories.SpotifyViewPlaylistFactory.create(type=ViewPlaylistTypeEnum.HH, market_code=market) factories.SpotifyViewPlaylistFactory.create(type=ViewPlaylistTypeEnum.NMF, market_code=f"mm_{i}") factories.SpotifyMarketRankFactory.create(type=MarketRegionTypeEnum.HH, market_code=market, rank=(i + 1)) factories.SpotifyMarketRankFactory.create(type=MarketRegionTypeEnum.NMF, market_code=market, rank=(i + 5)) for i in range(1, 20): for j, market in enumerate(streams_markets): factories.SpotifyAnalyticsAccountStreamInfoFactory.create( account=1, market=market, date=current_ts.date() - timedelta(days=i), total_streams=(1000 - j * 10 + i) ) index_module.update_ranks() assert get_db_value(db_session, "hh_rank_update_ts") == expected_ts.isoformat() db_ranks = ( db_session.query(SpotifyMarketRank) .filter(SpotifyMarketRank.type == MarketRegionTypeEnum.HH) .order_by(SpotifyMarketRank.rank) .all() ) assert {i.market_code: i.rank for i in db_ranks} == expected_ranks CURRENT_DATE = date(2022, 11, 10) def get_records( list_type: str, current_date: date, id_offset: int = 50, playlist_count: int = 2, items_count: int = 20, isrc_offset: int = 0, playlist_isrc_index_map: Dict[int, List[int]] = None, in_history: bool = False, with_id: bool = True, calculated_peak: bool = False, peak_index_list: List[int] = None, peak_date_diff_map: Dict[int, int] = None, peak_latest: bool = False, ) -> List[dict]: if not playlist_isrc_index_map: playlist_isrc_index_map = { i: [isrc_offset + i + j for j in range(1, items_count // playlist_count + 1)] for i in range(1, playlist_count + 1) } result = [] for i, (playlist_index, isrc_index_list) in enumerate(playlist_isrc_index_map.items()): playlist_id = f"pl_id_{playlist_index:02d}" for j, isrc_index in enumerate(isrc_index_list): isrc = f"isrc_{isrc_index:02d}" index = (playlist_index - 1) * (items_count // playlist_count + 1) + isrc_index - isrc_offset position = i * len(isrc_index_list) + j entry_date = ( current_date - timedelta(days=index % 3 + 1) if list_type == "change" else date(2022, 2, index % 20 + 1) ) latest_position = 20 + index latest_date = ( current_date - timedelta(days=1 if index % 2 and list_type == "change" else 0) if list_type in ("change", "new") else date(2022, 11, index % 4 + 5 - (4 if index % 4 in (0, 1) else 0)) ) peak_position = ( (playlist_index + isrc_index - 9) if calculated_peak else 10 + (peak_index_list[position] if peak_index_list else index) ) peak_date = ( (current_date - timedelta(days=isrc_index - 9)) if calculated_peak else ( current_date - timedelta(days=(peak_index_list[position] if peak_index_list else index) % 4) if list_type == "change" or peak_index_list else date(2022, 5, index % 15 + 1) ) ) result.append( { "playlist_id": playlist_id, "isrc": isrc, "entry_date": entry_date, "latest_date": latest_date, "latest_position": latest_position, "previous_date": date(2022, 11, index % 4 + 1), "previous_position": 15 + index, "peak_position": latest_position if peak_latest else peak_position, "peak_date": ( (latest_date if peak_latest else peak_date) + timedelta(days=peak_date_diff_map.get(playlist_index - 1, 0) if peak_date_diff_map else 0) ), "added_date": date(2022, 4, index % 30 + 1), "in_history": index % 3 != 0 or in_history, **({"id": id_offset + index} if with_id else {}), } ) return result def get_positions(playlist_id: str, isrc_list: List[str], current_date: date) -> dict: playlist_index = int(playlist_id.replace("spotify_pl_id_", "")) result = [] for isrc in isrc_list: isrc_index = int(isrc.replace("isrc_", "")) for i in range(1, 10): step = i + isrc_index - 10 result.append( { "playlist_id": playlist_id, "isrc": isrc, "date": (current_date - timedelta(days=step)).isoformat(), "current": playlist_index + step, } ) return {"items": result} def create_records(*args, **kwargs): record_list = get_records(*args, **kwargs) for item in record_list: factories.SpotifyHotHitsPlaylistTrackStatsFactory.create(**item) def records_to_response(record_list: List[dict]) -> dict: result_list = [] for item in record_list: result_list.append( { "playlist_id": item["playlist_id"], "isrc": item["isrc"], "is_top_track_for_isrc": True, "earliest_position_date_time": datetime.combine(item["entry_date"], time(14, 53, 21)).isoformat(), "date": item["latest_date"].isoformat(), "current": item["latest_position"], "last_date_change_14_days": item["previous_date"].isoformat(), "previous_position_change_14_days": item["previous_position"], "last_added_date_time": datetime.combine(item["added_date"], time(12, 2, 51)).isoformat(), "latest_position_date_time": datetime.combine(item["latest_date"], time(15, 7, 21)).isoformat(), } ) result_list.append( { "playlist_id": item["playlist_id"], "isrc": item["isrc"], "is_top_track_for_isrc": False, "earliest_position_date_time": datetime.combine( item["entry_date"] - timedelta(days=1), time(11, 42, 16) ).isoformat(), "date": item["latest_date"].isoformat(), "current": item["latest_position"] + 5, "last_date_change_14_days": (item["previous_date"] - timedelta(days=2)).isoformat(), "previous_position_change_14_days": item["previous_position"] + 10, "last_added_date_time": datetime.combine( item["added_date"] - timedelta(days=3), time(15, 26, 36) ).isoformat(), "latest_position_date_time": datetime.combine(item["latest_date"], time(7, 15, 46)).isoformat(), } ) return {"items": result_list} def object_to_dict(db_stats: List[SpotifyHotHitsPlaylistTrackStats]) -> List[dict]: return sorted( [{key: value for key, value in vars(i).items() if key not in ("_sa_instance_state", "id")} for i in db_stats], key=lambda i: (i["playlist_id"], i["isrc"]), ) def get_expected_result(): return ( get_records( list_type="old", current_date=CURRENT_DATE, playlist_count=4, items_count=20, isrc_offset=0, with_id=False ) + get_records( list_type="old", current_date=CURRENT_DATE, playlist_isrc_index_map={1: [12, 14], 2: [14, 15]}, calculated_peak=True, in_history=True, with_id=False, ) + get_records( list_type="new", current_date=CURRENT_DATE - timedelta(days=1), playlist_count=2, items_count=8, isrc_offset=13, playlist_isrc_index_map={1: [15], 2: [16]}, in_history=True, with_id=False, peak_index_list=[5, 12], peak_date_diff_map={0: 1, 1: 1}, ) + get_records( list_type="new", current_date=CURRENT_DATE, playlist_count=2, items_count=10, isrc_offset=14, playlist_isrc_index_map={1: [16], 2: [17]}, in_history=True, with_id=False, peak_index_list=[6, 13], ) + get_records( list_type="new", current_date=CURRENT_DATE, playlist_count=2, items_count=10, isrc_offset=14, playlist_isrc_index_map={1: [17, 18], 2: [18, 19]}, in_history=True, with_id=False, peak_latest=True, peak_date_diff_map={0: 0, 1: -1}, ) + get_records( list_type="new", current_date=CURRENT_DATE, playlist_count=2, items_count=10, isrc_offset=14, playlist_isrc_index_map={1: [19, 20], 2: [20, 21]}, in_history=True, with_id=False, peak_latest=True, ) ) def dsp_get(url: str, params: dict): start_date, end_date, playlist_id, isrc = ( params.get("start_date"), params.get("end_date"), params["playlist_id"], params.get("isrc") ) start_date = date.fromisoformat(start_date) if start_date else None end_date = date.fromisoformat(end_date) if end_date else None if not start_date and not end_date and not isrc: return records_to_response( get_records( list_type="new", current_date=CURRENT_DATE, playlist_count=2, items_count=10, isrc_offset=14 ) ) elif start_date and end_date and not isrc: return records_to_response( get_records( list_type="new", current_date=end_date, playlist_count=2, items_count=8, isrc_offset=13 ) ) elif start_date and end_date and isrc: return get_positions(playlist_id, isrc.split(","), CURRENT_DATE) elif not start_date and not end_date and isrc: return records_to_response( get_records( list_type="old", current_date=CURRENT_DATE, playlist_isrc_index_map={ int(playlist_id.replace("spotify_pl_id_", "")): [ int(i.replace("isrc_", "")) for i in isrc.split(",") ] }, ) ) def test_update_stats(mocker, db_session): config.UPDATE_PLAYLISTS_RANKS = False config.UPDATE_TABLE_LIST = [consts.Table.STATS] config.UPDATE_AGGREGATED_DATA = True create_records( list_type="old", current_date=CURRENT_DATE, id_offset=100, playlist_count=4, items_count=20, isrc_offset=0 ) create_records( list_type="change", current_date=CURRENT_DATE, id_offset=200, playlist_count=2, items_count=10, isrc_offset=10 ) for index in range(1, 5): factories.ApolloMarketFactory.create(id=index, code=f"m{index:02d}") factories.SpotifyViewPlaylistFactory.create( type="hh", playlist_id=f"pl_id_{index:02d}", market_code=f"m{index:02d}", last_date=CURRENT_DATE - timedelta(days=index), last_added_ts=datetime.combine(CURRENT_DATE - timedelta(days=1), time(15, 11, 47)), ) factories.ApolloKeyValueStorageFactory.create( key="hot_hits_latest_date", value=(CURRENT_DATE - timedelta(days=1)).isoformat() ) factories.ApolloKeyValueStorageFactory.create( key="hot_hits_latest_ts", value=(CURRENT_DATE - timedelta(days=1)).isoformat() + "21:12:11" ) factories.ApolloKeyValueStorageFactory.create( key="hh_job_last_run_ts", value=(CURRENT_DATE - timedelta(days=1)).isoformat() + "22:51:01" ) current_ts = datetime.utcnow() mocked_datetime = mocker.patch.object(index_module, "datetime", mocker.Mock(wraps=datetime)) mocked_datetime.utcnow.return_value = current_ts mocker.patch("dsp_api.get", side_effect=dsp_get) index_module.process_history_data() assert get_db_value(db_session, "hot_hits_latest_date") == CURRENT_DATE.isoformat() assert get_db_value(db_session, "hot_hits_latest_ts") == datetime.combine(CURRENT_DATE, time(15, 7, 21)).isoformat() assert get_db_value(db_session, "hh_job_last_run_ts") == current_ts.isoformat() expected_result = get_expected_result() db_stats = ( db_session.query(SpotifyHotHitsPlaylistTrackStats) .order_by(SpotifyHotHitsPlaylistTrackStats.playlist_id, SpotifyHotHitsPlaylistTrackStats.isrc) .all() ) assert object_to_dict(db_stats) == sorted(expected_result, key=lambda i: (i["playlist_id"], i["isrc"]))