from datetime import date, datetime, timedelta from distutils.util import strtobool from http import HTTPStatus from itertools import product import random from typing import Any, Dict, List, Optional, Tuple from unittest.mock import AsyncMock import pytest from server.cache.base import get_cache from server.cache.key_getters import get_key_by_specification from server.cache.utils import cached from server.client.clients.dsp_api import DspApiClient from server.client.clients.user_data_api import UserDataApiClient from server.constants.charts import NEW_ENTRY_TREND, ChartBreakdown, ChartType, YoutubeFullTrendMapping, \ YoutubeFullTrendInclude, YoutubeVideoImageSize, YoutubeVideoTrendField from server.scenarios.charts import youtube as scenarios from server.utils.pagination import multikeysort def get_charts_dates_result(*args, **kwargs) -> dict: result = { "charts": {"min": "2020-12-10", "max": "2021-04-10"}, "video": { "min": kwargs["min"] if "min" in kwargs else "2021-01-01", "max": kwargs["max"] if "max" in kwargs else "2021-03-19", }, "markets": [] if kwargs.get("no_markets") else ["cc1", "cc2", "cc3"], } return {k: v for k, v in result.items() if k in args} def get_dates(): return [{"earliest_data_date": "2020-12-10", "latest_data_date": "2021-04-10"}] @pytest.mark.parametrize( "params,expected_result,call_count,enable_cache,no_markets,no_video", ( ({}, {"charts": {"min": "2020-12-10", "max": "2021-04-10"}}, 1, False, False, False), ({"market": "us"}, get_charts_dates_result("charts"), 1, False, False, False), ({"video_id": "v_id_1"}, get_charts_dates_result("charts", "video"), 2, False, False, False), ({"video_id": "v_2"}, get_charts_dates_result("charts", "video", min=None, max=None), 2, False, True, True), ( {"video_id": "v_id_1", "market": "us"}, get_charts_dates_result("charts", "video", min="2021-01-09", max="2021-03-11"), 2, False, False, False, ), ({"include": "markets"}, get_charts_dates_result("charts", "markets"), 2, False, False, False), ({"include": "markets"}, get_charts_dates_result("charts", "markets", no_markets=True), 2, False, True, True), ( {"video_id": "v_id_1", "include": "markets"}, get_charts_dates_result("charts", "markets", "video"), 3, True, False, False, ), ), ) async def test_get_youtube_charts_dates( params, expected_result, call_count, enable_cache, no_markets, no_video, mocker, auth, client ): async def dsp_send(url: str, *args, **kwargs): if url == "api/delphi/charts": return [] if no_markets else [{"country_code": f"cc{i}"} for i in range(1, 4)] elif url == "api/delphi/data-health/chart-positions": return get_dates() elif url == "api/delphi/video-positions/charts/summary": if no_video: return [] return [ {"summary": {"earliest_position_date": f"2021-01-0{10 - i}", "latest_position_date": f"2021-03-1{i}"}} for i in range(1, 2 if "country_code" in kwargs["params"] else 10) ] scenarios.get_markets = cached(ttl=(6000 if enable_cache else 0))(scenarios.get_markets.__wrapped__) dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get("/api/charts/youtube/dates/", headers=auth, params=params) assert response.status == HTTPStatus.OK assert dsp_send_mock.call_count == call_count response = await response.json() assert response == expected_result cache = get_cache() key = get_key_by_specification(scenarios.get_markets, (), {}) value = await cache.get(key) if enable_cache: assert value == expected_result["markets"] else: assert value is None def get_youtube_charts_data(include: List[str] or str = None, offset: int = 2, limit: int = 8, **kwargs) -> List[dict]: if isinstance(include, str): include = [include] if kwargs: min_date = date(2021, 3, 10) max_date = min_date + timedelta(days=9) start_date, end_date = ( date.fromisoformat(kwargs["params"]["start_date"]), date.fromisoformat(kwargs["params"]["end_date"]), ) offset = offset if start_date <= min_date else (start_date - min_date).days limit = limit if end_date >= max_date else (end_date - max_date).days + limit return [ { "chart": {"chart_id": "ch_id_1", "name": "ch_name_1", "country_code": f"c{i}"}, "video_id": f"v_id_{i}", **( { "position": { "current": i * 2 + 5, "date": f"2021-03-1{i}", "is_entry": bool(i % 4), "is_reentry": bool((i + 1) % 4), "trend": i - 4, }, } if "position" in include or not include else {} ), **( { "summary": { "earliest_position_date": f"2021-02-2{i}", "min_position": i + 10, "min_position_date": f"2021-05-1{i}", "total_days": i * 2 + 20, } } if "summary" in include or not include else {} ), **( { "video": { "video_title": f"v_t_{i}", "channel_id": f"ch_id_{i}", "channel_title": f"ch_t_{i}", "content_type": f"ct_{i}", "date_uploaded": f"2021-03-2{i}", "thumbnails": { "default": {"url": f"d_url_{i}"}, "medium": {"url": f"m_url_{i}"}, } } } if "video" in include or not include else {} ), } for i in range(offset, limit) ] def filter_by_include( include: List[YoutubeVideoTrendField], mapping: Dict[YoutubeVideoTrendField, Any] ) -> Dict[str, Any] or None: if not mapping: return None return {i.value: mapping.get(i) for i in include} def get_youtube_charts_result( include: List[YoutubeVideoTrendField] = None, offset: int = 1, limit: int = 10, position_range: Tuple[int, int] = (2, 6), summary_range: Tuple[int, int] = (3, 6), revert: bool = False, positions: List[int] = None, ): if include is None: include = list(YoutubeVideoTrendField) return [ { "country_code": f"c{i}", "data": filter_by_include( include, { **( { YoutubeVideoTrendField.CURRENT_POSITION: i * 2 + 5, YoutubeVideoTrendField.DATE: f"2021-03-1{i}", YoutubeVideoTrendField.IS_NEW: bool(i % 4), YoutubeVideoTrendField.IS_RE_ENTRY: bool((i + 1) % 4), YoutubeVideoTrendField.TREND: NEW_ENTRY_TREND if i % 4 or (i + 1) % 4 else 4 - i, } if i in range(position_range[0], position_range[1]) else {} ), **( { YoutubeVideoTrendField.ADDED_DATE: f"2021-02-2{i}", YoutubeVideoTrendField.PEAK_POSITION: i + 10, YoutubeVideoTrendField.PEAK_POSITION_DATE: f"2021-05-1{i}", YoutubeVideoTrendField.IN_CHART_DAYS: i * 2 + 20, } if i in range(summary_range[0], summary_range[1]) else {} ), }, ), } for i in (positions if positions else (range(limit - 1, offset - 1, -1) if revert else range(offset, limit))) ] @pytest.mark.parametrize( "params,status,expected_result,call_count", ( ({}, HTTPStatus.BAD_REQUEST, None, 0), ({"video_id": "vi_8", "fields": "peak_position,date", "sort_by": "trend"}, HTTPStatus.BAD_REQUEST, None, 0), ({"video_id": "vi_8", "sort_by": "date"}, HTTPStatus.BAD_REQUEST, None, 0), ({"video_id": "vi_1"}, HTTPStatus.OK, get_youtube_charts_result(), 4), ({"video_id": "vi_1", "all_markets": "false"}, HTTPStatus.OK, get_youtube_charts_result(offset=2, limit=8), 3), ({"video_id": "vi_2", "date": "2021-05-01"}, HTTPStatus.OK, get_youtube_charts_result(), 3), ({"video_id": "vi_3", "sort_order": "desc"}, HTTPStatus.OK, get_youtube_charts_result(revert=True), 4), ( {"video_id": "vi_3", "sort_by": "trend"}, HTTPStatus.OK, get_youtube_charts_result(positions=[2, 3, 4, 5, 1, 6, 7, 8, 9]), 4, ), ( {"video_id": "vi_1", "date": "2021-05-01", "all_markets": "false", "fields": "trend"}, HTTPStatus.OK, get_youtube_charts_result(include=[YoutubeVideoTrendField.TREND], offset=2, limit=6), 1, ), ( {"video_id": "vi_1", "all_markets": "false", "fields": "peak_position"}, HTTPStatus.OK, get_youtube_charts_result( include=[YoutubeVideoTrendField.PEAK_POSITION], offset=3, limit=8, summary_range=(3, 8), ), 2, ), ( {"video_id": "vi_2", "fields": "peak_position,trend,date", "sort_by": "trend", "sort_order": "desc"}, HTTPStatus.OK, get_youtube_charts_result( include=[ YoutubeVideoTrendField.PEAK_POSITION, YoutubeVideoTrendField.TREND, YoutubeVideoTrendField.DATE, ], positions=[1, 6, 7, 8, 9, 2, 3, 4, 5] ), 4, ), ( {"video_id": "vi", "all_markets": "false", "fields": "is_new,is_re_entry,trend", "sort_by": "country_code"}, HTTPStatus.OK, get_youtube_charts_result( include=[ YoutubeVideoTrendField.IS_NEW, YoutubeVideoTrendField.IS_RE_ENTRY, YoutubeVideoTrendField.TREND, ], offset=2, limit=6, ), 2, ), ), ) async def test_get_youtube_video_trend(params, status, expected_result, call_count, mocker, auth, client): async def dsp_send(url: str, *args, **kwargs): if url == "api/delphi/data-health/chart-positions": return get_dates() elif url == "api/delphi/video-positions/charts": return get_youtube_charts_data("position", limit=6) elif url == "api/delphi/video-positions/charts/summary": return get_youtube_charts_data("summary", offset=3) elif url == "api/delphi/charts": return [{"country_code": f"c{i}"} for i in range(1, 10)] dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get("/api/charts/youtube/video-trend/", headers=auth, params=params) assert response.status == status assert dsp_send_mock.call_count == call_count if status == HTTPStatus.OK: response = await response.json() assert response == expected_result def get_yvp_result(offset: int = 0, limit: int = 10, include_chart: bool = False): return { **( { "chart": { "chart_id": "ch_id_1", "name": "ch_name_1", "country_code": f"c{offset}", } } if include_chart else {} ), "items": [ {"date": f"2021-03-1{i}", "position": i * 2 + 5} for i in range(offset, limit) ], "count": limit - offset, } @pytest.mark.parametrize( "params,status,expected_result,call_count", ( ({}, HTTPStatus.BAD_REQUEST, None, 0), ({"video_id": "v_id_5"}, HTTPStatus.BAD_REQUEST, None, 0), ({"video_id": "v_id_1", "market": "us"}, HTTPStatus.OK, get_yvp_result(offset=3), 2), ( {"video_id": "v_id_3", "market": "gb", "include": "chart"}, HTTPStatus.OK, get_yvp_result(offset=3, include_chart=True), 2, ), ( {"video_id": "v_id_2", "market": "ca", "start_date": "2021-03-14", "end_date": "2021-03-17"}, HTTPStatus.OK, get_yvp_result(offset=4, limit=8), 1, ), ( { "video_id": "v_id", "market": "global", "include": "chart", "start_date": "2021-03-01", "end_date": "2021-03-15", }, HTTPStatus.OK, get_yvp_result(limit=6, include_chart=True), 1, ), ( {"video_id": "v_id_8", "market": "ca", "start_date": "2021-03-01", "end_date": "2021-03-05"}, HTTPStatus.OK, get_yvp_result(limit=0), 1, ), ), ) async def test_get_youtube_video_positions(params, status, expected_result, call_count, mocker, auth, client): async def dsp_send(url: str, *args, **kwargs): if url == "api/delphi/data-health/chart-positions": return get_dates() elif url == "api/delphi/video-positions/charts": return get_youtube_charts_data("position", offset=0, limit=10, **kwargs) dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get("/api/charts/youtube/video-positions/", headers=auth, params=params) assert response.status == status assert dsp_send_mock.call_count == call_count if status == HTTPStatus.OK: response = await response.json() assert response == expected_result def get_full_trend_result( include: List[YoutubeFullTrendInclude] or YoutubeFullTrendInclude or None = None, offset: int = 2, limit: int = 8, summary_range: Tuple[int, int] = (3, 8), image_size: YoutubeVideoImageSize = YoutubeVideoImageSize.DEFAULT, ) -> List[dict]: if isinstance(include, YoutubeFullTrendInclude): include = [include] if not include: include = list(YoutubeFullTrendInclude) summary_range = range(summary_range[0], summary_range[1]) result = [] for index in range(offset, limit): result_item = {"video_id": f"v_id_{index}"} data = get_youtube_charts_data([i.value for i in include], index, index + 1)[0] for data_type in include: mapping = YoutubeFullTrendMapping[data_type] data_item = ( {} if data_type == YoutubeFullTrendInclude.SUMMARY and index not in summary_range else data[data_type.value] ) result_item[data_type.value] = { k: ( ( NEW_ENTRY_TREND if data_item.get("is_entry") or data_item.get("is_reentry") else data_item.get(v, 0) * -1 ) if k == "trend" else data_item.get(v) ) for k, v in mapping.items() } if data_type == YoutubeFullTrendInclude.VIDEO: result_item[YoutubeFullTrendInclude.VIDEO.value]["image_url"] = ( data[YoutubeFullTrendInclude.VIDEO.value]["thumbnails"].get(image_size.value, {}).get("url") ) result.append(result_item) return result @pytest.mark.parametrize( "params,status,expected_result,call_count,position_data,summary_data", ( ({}, HTTPStatus.BAD_REQUEST, None, 0, [], []), ({"market": "us"}, HTTPStatus.BAD_REQUEST, None, 0, [], []), ({"date": "2021-03-14"}, HTTPStatus.BAD_REQUEST, None, 0, [], []), ({"market": "us", "date": "2021-03-14", "image_size": "unknown"}, HTTPStatus.BAD_REQUEST, None, 0, [], []), ( {"market": "us", "date": "2021-03-14"}, HTTPStatus.OK, get_full_trend_result(), 2, get_youtube_charts_data(["position", "video"]), get_youtube_charts_data("summary", offset=3), ), ( {"market": "us", "date": "2021-03-14"}, HTTPStatus.OK, [], 1, [], get_youtube_charts_data("summary", offset=3), ), ( {"market": "us", "date": "2021-03-14"}, HTTPStatus.OK, get_full_trend_result(summary_range=(0, 0)), 2, get_youtube_charts_data(["position", "video"]), [], ), ( {"market": "us", "date": "2021-03-14", "image_size": "medium"}, HTTPStatus.OK, get_full_trend_result(image_size=YoutubeVideoImageSize.MEDIUM), 2, get_youtube_charts_data(["position", "video"]), get_youtube_charts_data("summary", offset=3), ), ( {"market": "us", "date": "2021-03-14", "image_size": "high"}, HTTPStatus.OK, get_full_trend_result(image_size=YoutubeVideoImageSize.HIGH), 2, get_youtube_charts_data(["position", "video"]), get_youtube_charts_data("summary", offset=3), ), ( {"market": "us", "date": "2021-03-14", "include": "summary"}, HTTPStatus.OK, get_full_trend_result(YoutubeFullTrendInclude.SUMMARY), 2, get_youtube_charts_data(["position", "video"]), get_youtube_charts_data("summary", offset=3), ), ( {"market": "us", "date": "2021-03-14", "include": "position,video"}, HTTPStatus.OK, get_full_trend_result([YoutubeFullTrendInclude.POSITION, YoutubeFullTrendInclude.VIDEO]), 1, get_youtube_charts_data(["position", "video"]), get_youtube_charts_data("summary", offset=3), ), ( {"market": "us", "date": "2021-03-14", "include": "video"}, HTTPStatus.OK, get_full_trend_result(YoutubeFullTrendInclude.VIDEO), 1, get_youtube_charts_data(["position", "video"]), get_youtube_charts_data("summary", offset=3), ), ), ) async def test_get_youtube_full_trend( params, status, expected_result, call_count, position_data, summary_data, mocker, auth, client ): async def dsp_send(url: str, *args, **kwargs): if url == "api/delphi/video-positions/charts/summary": return summary_data elif url == "api/delphi/video-positions/charts": return position_data dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get("/api/charts/youtube/full-trend/", headers=auth, params=params) assert response.status == status assert dsp_send_mock.call_count == call_count if status == HTTPStatus.OK: response = await response.json() assert response == expected_result def get_starred(items: List[int] = None, count: int = 5, starred: List[int] = None) -> List[dict]: if not items: items = list(range(1, count + 1)) result = [] for i in items: if starred and i in starred or not starred and bool(i % 2): result.append({ "entity_id": f"isrc_{i}", "favorites_id": i }) return result def get_sony(items: List[int] = None, count: int = 5, licensors: bool = False) -> dict: items_set = bool(items) if not items: items = list(range(1, count + 1)) return {f"tr_{i}": "sme" for i in items if items_set or (not i % 2 if licensors else bool(i % 3))} def get_date(chart_date: date, start: int, end: int) -> str: return (chart_date - timedelta(days=random.randint(start, end))).isoformat() def get_chart_date( market_list: List[str] = None, market_mapping: Optional[Dict[str, date]] = None, dsp: str = "spotify", chart_date: Optional[date or str] = None, breakdowns: Optional[List[str] or str] = ChartBreakdown.DAILY.value, types: Optional[List[str] or str] = ChartType.REGIONAL.value, ) -> dict: if market_mapping: market_list = list(market_mapping.keys()) else: if not chart_date: chart_date = datetime.now().date() if not isinstance(chart_date, str): chart_date = chart_date.isoformat() if dsp == "spotify": if not breakdowns: breakdowns = ChartBreakdown.values() elif isinstance(breakdowns, str): breakdowns = [breakdowns] if not types: types = ChartType.values() elif isinstance(types, str): types = [types] return { "breakdown_chart_type_country_code": { breakdown_name: { type_name: { market: { "max_date": market_mapping[market] if market_mapping else chart_date, } for market in market_list } for type_name in types } for breakdown_name in breakdowns } } else: return { "dsp_chart_type_country_code": { "apple": { "charts_daily": { market: { "max_date": market_mapping[market] if market_mapping else chart_date, "last_update_date_time": market_mapping[market] if market_mapping else chart_date } for market in market_list } } } } def get_chart(items: List[int] = None, count: int = 5, chart_date: Optional[date or str] = None) -> dict: if isinstance(chart_date, str): chart_date = date.fromisoformat(chart_date) if not chart_date: chart_date = datetime.now().date() previous_date = chart_date - timedelta(days=1) if not items: items = list(range(1, count + 1)) return { "items": [ { "licensors": ["sme"] if not i % 2 else [], "metrics": { "date": chart_date.isoformat(), "previous_date": previous_date.isoformat(), "is_entry": (i % 4) == 1, "position": i, "previous_position": 0 if (i % 4) == 1 else i + i % 4 - 2 + random.randint(0, 20), "date_streams": random.randint(1, 999999), }, "lifetime_metrics": { "earliest_position": random.randint(1, 200), "earliest_position_date": get_date(chart_date, 0, 10), "latest_position": random.randint(1, 200), "latest_position_date": get_date(chart_date, 0, 5), "max_position": random.randint(1, 200), "max_position_date": get_date(chart_date, 0, 10), "min_position": random.randint(1, 200), "min_position_date": get_date(chart_date, 0, 10), "total_days": random.randint(1, 300), }, "public_meta": { "track_id": f"tr_{i}", "isrc": f"isrc_{i}", "artists": [ {"artist_id": f"ar_{i}_{j}", "name": f"a_name_{i}_{j}"} for j in range(random.randint(1, 4)) ], "uri": f"https://open.spotify.com/track/tr_{i}", "release_date": get_date(chart_date, 0, 999), "image_url": f"https://i.scdn.co/image/img_{i}", "name": f"t_name_{i}", } } for i in items ] } def get_c_result( dsp: str, url: str, items: List[dict], starred_data: list or None, sony_data: dict or None, is_available: bool, **kwargs, ): position_max = 201 if dsp == "spotify" else 101 def get_change(i: dict) -> int: return ( (position_max - i["metrics"]["position"]) if url == "removals" else (i["metrics"]["position"] - (i["metrics"]["previous_position"] or position_max)) ) sony_data = [t_id for t_id, dist in sony_data.items() if dist == "sme"] if dsp == "spotify" and sony_data else [] starred_map = {t["entity_id"]: t["favorites_id"] for t in (starred_data or [])} chart_date = ( ( date.fromisoformat(items[0]["metrics"]["date"]) + timedelta(days=1 if kwargs.get("type") == "weekly" or url == "removals" else 0) ).isoformat() if items else datetime.now().date().isoformat() ) if kwargs.get("start") and kwargs.get("end") and url == "out": items = [ i for i in items if ( kwargs["start"] <= i["metrics"]["previous_position"] <= kwargs["end"] and (i["metrics"]["position"] > kwargs["end"] or i["metrics"]["position"] < kwargs["start"]) ) ] if "change" in kwargs: items = [ i for i in items if ( i["metrics"]["previous_position"] and abs(i["metrics"]["position"] - i["metrics"]["previous_position"]) >= kwargs["change"] ) ] result = { "available": is_available, "chart_date": chart_date, "tracks": [ { "is_re_enter": ( i["metrics"]["is_entry"] and url != "removals" and i["lifetime_metrics"]["earliest_position_date"] != i["metrics"]["date"] ), "image_url": i["public_meta"]["image_url"], "name": i["public_meta"]["name"], "chart_date": chart_date, "distributed_by": "sme" if i["public_meta"]["track_id"] in sony_data else None, "is_starred_track": i["public_meta"]["isrc"] in starred_map, "favorites_id": starred_map.get(i["public_meta"]["isrc"]), "change": get_change(i), "artist": ", ".join(j["name"] for j in i["public_meta"]["artists"]), "streams": i["metrics"]["date_streams"], "id": i["public_meta"]["track_id"], "is_sony": i["public_meta"]["track_id"] in sony_data if sony_data is not None else False, "isrc": i["public_meta"]["isrc"], "is_new": i["metrics"]["is_entry"] and url != "removals", "artists": [ { "name": j["name"], "uri": "spotify:artist:" + j["artist_id"], "id": j["artist_id"], } for j in i["public_meta"]["artists"] ], "position": i["metrics"]["position"], "trend": get_change(i) - (1000 if i["metrics"]["is_entry"] and url != "removals" else 0), } for i in items ], } if kwargs.get("is_sony") is not None: is_sony = strtobool(kwargs["is_sony"]) result["tracks"] = [i for i in result["tracks"] if i["is_sony"] == is_sony] if strtobool(kwargs.get("only_starred_tracks", "false")): result["tracks"] = [i for i in result["tracks"] if i["is_starred_track"]] result["tracks"] = multikeysort(result["tracks"], kwargs.get("order_by", "position").split(",")) if url == "moves": result.update({"count": len(result["tracks"])}) result["tracks"] = result["tracks"][kwargs.get("offset", 0): kwargs.get("limit", 200) + kwargs.get("offset", 0)] return result @pytest.mark.parametrize( "url,params,status,chart_data,starred_data,sony_data,is_available,call_count", ( ("top", {}, HTTPStatus.BAD_REQUEST, [], {}, [], True, (0, 0, 0)), ("top", {"vendor": "spotify"}, HTTPStatus.BAD_REQUEST, [], {}, [], True, (0, 0, 0)), ( "top", {"vendor": "spotify", "market": "us"}, HTTPStatus.OK, get_chart(count=0), {}, [], False, (3, 0, 0), ), ( "top", {"vendor": "spotify", "market": "us"}, HTTPStatus.OK, get_chart(), None, None, True, (3, 0, 0), ), ( "top", {"vendor": "spotify", "market": "us", "fields": "is_sony"}, HTTPStatus.OK, get_chart(), None, get_sony(), True, (3, 0, 1), ), ( "top", {"vendor": "spotify", "market": "us", "fields": "is_sony", "is_sony": "true"}, HTTPStatus.OK, get_chart(), None, get_sony(), True, (3, 0, 1), ), ( "top", {"vendor": "spotify", "market": "us", "fields": "is_starred"}, HTTPStatus.OK, get_chart(), get_starred(), None, True, (3, 1, 0), ), ( "top", {"vendor": "spotify", "market": "us", "fields": "is_sony"}, HTTPStatus.OK, get_chart(), None, get_sony(licensors=True), True, (3, 0, 1), ), ( "top", {"vendor": "spotify", "market": "us", "fields": "is_sony,is_starred"}, HTTPStatus.OK, get_chart(), get_starred(), get_sony(licensors=True), True, (3, 1, 1), ), ( "top", {"vendor": "spotify", "market": "us", "fields": "is_starred", "only_starred_tracks": "true"}, HTTPStatus.OK, get_chart(), get_starred(), None, True, (3, 1, 0), ), ( "top", { "vendor": "spotify", "market": "global", "fields": "is_starred,image_url,chart_date,trends,artists,is_re_enter,is_sony", "type": "weekly", "date": "2021-10-22", "order_by": "trend,-name", }, HTTPStatus.OK, get_chart(chart_date="2021-10-21"), get_starred(), get_sony(), True, (2, 1, 1), ), ( "top", { "vendor": "spotify", "market": "global", "fields": "is_starred,image_url,chart_date,trends,artists,is_re_enter,is_sony", "type": "weekly", "date": "2021-10-22", "order_by": "trend,-name", "start": 2, "end": 3, }, HTTPStatus.OK, get_chart(items=[2, 3], chart_date="2021-10-21"), get_starred(), get_sony(), True, (2, 1, 1), ), ( "out", { "vendor": "spotify", "market": "worldwide", "fields": "is_starred,image_url,chart_date,trends,artists,is_re_enter,is_sony", "order_by": "name,-trend", "start": 10, "end": 30, }, HTTPStatus.OK, get_chart(), get_starred(), get_sony(), True, (3, 1, 1), ), ( "additions", { "vendor": "spotify", "market": "gb", "fields": "is_starred,chart_date,trends,artists,is_sony", "order_by": "-id", }, HTTPStatus.OK, get_chart(), get_starred(), get_sony(), True, (3, 1, 1), ), ( "removals", { "vendor": "spotify", "market": "ca", "fields": "is_starred,trends,is_sony", "order_by": "id", }, HTTPStatus.OK, get_chart(), get_starred(), get_sony(), True, (3, 1, 1), ), ( "moves", { "vendor": "spotify", "market": "de", "fields": "is_starred,is_sony", "order_by": "-is_starred_track,is_sony", "change": 2, }, HTTPStatus.OK, get_chart(count=10), get_starred(count=10), get_sony(), True, (3, 1, 1), ), ( "moves", { "vendor": "spotify", "market": "de", "fields": "is_starred,is_sony", "order_by": "-is_starred_track,is_sony", "change": 2, "offset": 2, "limit": 3, }, HTTPStatus.OK, get_chart(count=10), get_starred(count=10), get_sony(), True, (3, 1, 1), ), ( "top", { "vendor": "apple", "market": "us", "fields": "is_starred,image_url,chart_date,trends,artists,is_re_enter,is_sony", "order_by": "-is_starred_track,is_sony", }, HTTPStatus.OK, get_chart(), get_starred(), {}, True, (3, 1, 1), ), ( "top", { "vendor": "spotify", "market": "global", "fields": "is_sony", "type": "daily", "date": "2021-10-22", "is_sony": "false", "start": 1, "end": 3, }, HTTPStatus.OK, get_chart(items=[1, 2, 3], chart_date="2021-10-21"), None, get_sony(), True, (2, 0, 1), ), ( "top", {"vendor": "apple", "market": "us", "date": "2022-05-01"}, HTTPStatus.OK, get_chart(), None, None, True, (2, 0, 0), ), ), ) async def test_get_chart_digest( url: str, params: dict, status: int, chart_data: dict, starred_data: list or None, sony_data: dict or None, is_available: bool, call_count: Tuple[int, int, int], mocker, auth, client, ): async def dsp_send(url: str, *args, **kwargs): if url == "api/delphi/spotify/charts": return {"items": [{"id": 1}] if is_available else []} elif url.find("charts/data-health/status") >= 0: return get_chart_date(market_list=[params["market"]], dsp=params.get("vendor", "spotify")) else: return chart_data dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) async_ud_mock = AsyncMock(return_value={"items": starred_data}) get_starred_mock = mocker.patch.object(UserDataApiClient, "_send_request", side_effect=async_ud_mock) is_sony_mock = AsyncMock(return_value=sony_data) is_sony_func_mock = mocker.patch( "server.scenarios.charts.digest.get_tracks_distributors_map", side_effect=is_sony_mock) response = await client.get(f"/api/charts/{url}/", headers=auth, params=params) assert response.status == status dsp_call, apollo_call, sony_call = call_count assert dsp_send_mock.call_count == dsp_call assert get_starred_mock.call_count == apollo_call assert is_sony_func_mock.call_count == sony_call if status == HTTPStatus.OK: response = await response.json() if url == "moves": del response["next"] del response["previous"] assert response == get_c_result( params.get("vendor", "spotify"), url, chart_data["items"], starred_data, sony_data, is_available, **params) @pytest.mark.parametrize( "params,status,expected_result", ( ({}, HTTPStatus.BAD_REQUEST, None), ({"vendor": "spotify"}, HTTPStatus.BAD_REQUEST, None), ( {"vendor": "spotify", "market": "us"}, HTTPStatus.OK, {"max_date": "2021-11-02", "min_date": "2017-01-01"}, ), ( {"vendor": "apple", "market": "worldwide"}, HTTPStatus.OK, {"max_date": "2021-11-03", "min_date": "2017-01-01"}, ), ( {"vendor": "spotify", "market": "worldwide", "type": "weekly"}, HTTPStatus.OK, {"max_date": "2021-11-01", "min_date": "2016-12-29"}, ), ), ) async def test_get_date_range(params: dict, status: HTTPStatus, expected_result: dict, mocker, auth, client): test_data = {"spotify": {"worldwide": "2021-11-01", "us": "2021-11-02"}, "apple": {"worldwide": "2021-11-03"}} current_date = ( test_data[params.get("vendor")][params.get("market")] if params.get("vendor") and params.get("market") else None ) async_dsp_mock = AsyncMock( return_value=( get_chart_date(market_list=[params["market"]], chart_date=current_date, dsp=params["vendor"]) if current_date else None ) ) dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=async_dsp_mock) response = await client.get(f"/api/charts/date-range/", headers=auth, params=params) assert response.status == status if status == HTTPStatus.OK: assert dsp_send_mock.call_count == 1 response = await response.json() assert response == expected_result @pytest.mark.parametrize( "params,status,expected_result,call_count", ( ({}, HTTPStatus.BAD_REQUEST, None, 0), ( {"isrc": "AAA123456789"}, HTTPStatus.OK, [{"current_position": 41, "entry_date": "2020-02-01", "isrc": "AAA123456789", "peak_position": 11}], 2, ), ( { "isrc": "AAA123456789,BBB987654321", "list_type": "viral", "market": "us", "type": "weekly", "vendor": "apple", }, HTTPStatus.OK, [ {"current_position": 41, "entry_date": "2020-02-01", "isrc": "AAA123456789", "peak_position": 11}, {"current_position": None, "entry_date": "2020-02-02", "isrc": "BBB987654321", "peak_position": 12}, ], 2, ), ), ) async def test_get_stats(params: dict, status: int, expected_result: list, call_count: int, mocker, auth, client): latest_date = "2021-11-11" isrc_list = params["isrc"].split(",") if "isrc" in params else [] async def dsp_send(url: str, *args, **kwargs): dsp = "spotify" if "spotify" in url else "apple" relative_url = url.replace("api/delphi/spotify/", "").replace("api/delphi/apple-music/", "") params = kwargs["params"] if relative_url == "charts/data-health/status": return get_chart_date(market_list=params["country_code"], dsp=dsp, chart_date=latest_date) elif relative_url == "tracks/charts/lifetime": chart_type, breakdown, country_code, isrc = ( params.get("chart_type", "charts"), params.get("chart_breakdown", "daily"), params["chart_country_code"], params["isrc"], ) return { "items": [ { "chart_meta": {"chart_id": f"{chart_type}_{breakdown}_{country_code}"}, "public_meta": {"isrc": isrc}, "lifetime_metrics": { "earliest_position_date": f"2020-02-0{index + 1}", "min_position": 11 + index, "latest_position": 41 + index, "latest_position_date": f"2021-11-1{index + 1}", }, } for index, isrc in enumerate(isrc_list) ] } dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get(f"/api/charts/stats/", headers=auth, params=params) assert response.status == status if status == HTTPStatus.OK: assert dsp_send_mock.call_count == call_count response = await response.json() assert sorted(response, key=lambda i: i["isrc"]) == expected_result @pytest.mark.parametrize( "params,status,expected_result,call_count", ( ({}, HTTPStatus.BAD_REQUEST, None, (0, 0)), ({"isrc": "AAA123456789"}, HTTPStatus.BAD_REQUEST, None, (0, 0)), ({"vendor": "spotify"}, HTTPStatus.BAD_REQUEST, None, (0, 0)), ( {"isrc": "AAA123456789", "vendor": "spotify", "sort_by": "peak_position"}, HTTPStatus.OK, { "count": 3, "items": [ { "added_date": "2020-02-01", "change": -8, "country_code": "m9", "country_name": None, "date": "2021-12-08", "is_new": False, "is_re_enter": False, "peak_position": 19, "position": 39, "rank": 9, "streams": 1009, }, { "added_date": "2020-02-02", "change": None, "country_code": "m10", "country_name": "Market 10", "date": "2021-12-08", "is_new": False, "is_re_enter": True, "peak_position": 20, "position": 40, "rank": 10, "streams": 1010, }, { "added_date": "2020-02-03", "change": None, "country_code": "m11", "country_name": "Market 11", "date": "2021-12-08", "is_new": False, "is_re_enter": True, "peak_position": 21, "position": 41, "rank": 11, "streams": 1011, }, ], "next": None, "previous": None, "top_market": "m9", }, (2, 1), ), ( {"isrc": "AAA123456789", "vendor": "spotify", "search": "market 11"}, HTTPStatus.OK, { "count": 2, "items": [ { "added_date": "2020-02-03", "change": None, "country_code": "m11", "country_name": "Market 11", "date": "2021-12-08", "is_new": False, "is_re_enter": True, "peak_position": 21, "position": 41, "rank": 11, "streams": 1011, }, { "added_date": "2020-02-01", "change": -8, "country_code": "m9", "country_name": None, "date": "2021-12-08", "is_new": False, "is_re_enter": False, "peak_position": 19, "position": 39, "rank": 9, "streams": 1009, }, ], "next": None, "previous": None, "top_market": "m9", }, (2, 1), ), ( {"isrc": "AAA123456789", "vendor": "spotify", "search": "market 11", "limit": 1, "offset": 1}, HTTPStatus.OK, { "count": 2, "items": [ { "added_date": "2020-02-01", "change": -8, "country_code": "m9", "country_name": None, "date": "2021-12-08", "is_new": False, "is_re_enter": False, "peak_position": 19, "position": 39, "rank": 9, "streams": 1009, }, ], "next": None, "previous": "/gate-api/charts/summary/?isrc=AAA123456789&vendor=spotify&search=market+11&limit=1", "top_market": "m9", }, (2, 1), ), ( {"isrc": "qwe123456789", "vendor": "apple", "limit": 1, "offset": 1}, HTTPStatus.OK, { "count": 3, "items": [ { "added_date": "2020-02-03", "change": None, "country_code": "m11", "country_name": "Market 11", "date": date.today().isoformat(), "is_new": False, "is_re_enter": True, "peak_position": 21, "position": 41, "rank": 11, "streams": 1011, }, ], "next": "/gate-api/charts/summary/?isrc=qwe123456789&vendor=apple&limit=1&offset=2", "previous": "/gate-api/charts/summary/?isrc=qwe123456789&vendor=apple&limit=1", "top_market": "m9", }, (3, 1), ), ( {"isrc": "RTY123456789", "vendor": "apple", "sort_by": "added_date"}, HTTPStatus.OK, { "count": 3, "items": [ { "added_date": "2020-02-01", "change": -8, "country_code": "m9", "country_name": None, "date": date.today().isoformat(), "is_new": False, "is_re_enter": False, "peak_position": 19, "position": 39, "rank": 9, "streams": 1009 }, { "added_date": "2020-02-02", "change": None, "country_code": "m10", "country_name": "Market 10", "date": date.today().isoformat(), "is_new": False, "is_re_enter": True, "peak_position": 20, "position": 40, "rank": 10, "streams": 1010 }, { "added_date": "2020-02-03", "change": None, "country_code": "m11", "country_name": "Market 11", "date": date.today().isoformat(), "is_new": False, "is_re_enter": True, "peak_position": 21, "position": 41, "rank": 11, "streams": 1011 } ], "next": None, "previous": None, "top_market": "m9" }, (3, 1), ), ), ) async def test_get_summary( params: dict, status: int, expected_result: list, call_count: Tuple[int, int], mocker, auth, client ): async def dsp_send(url: str, *args, **kwargs): relative_url = url.replace("api/delphi/spotify/", "").replace("api/delphi/apple-music/", "") if relative_url == "tracks/charts": return { "items": [ { "chart_meta": {"chart_id": f"t1_b1_m{i}", "country_code": f"m{i}", "rank": i}, "metrics": { "date": "2021-12-08", "position": 30 + i, "previous_position": 40 - i, "date_streams": 1000 + i, "is_entry": bool(i % 3), }, "lifetime_metrics": { "earliest_position_date": f"2020-02-0{i % 9 + 1}", "min_position": 10 + i, "latest_position": 20 + i, "latest_position_date": (date(2021, 12, 9) - timedelta(days=i // 3)).isoformat(), }, } for i in range(9, 12) ] } elif relative_url == "api/delphi/charts": return [{"chart_id": f"ch_id_{i}", "country_code": f"m{i}"} for i in (4, 8, 2, 5, 9, 10)] elif relative_url == "api/delphi/regions": return {"items": [{"country_code": f"m{i}", "country_name": f"Market {i}"} for i in range(10, 30)]} elif relative_url == "charts/data-health/status": markets = { f"m{i}": { "last_update_date_time": datetime.today().isoformat() } for i in range(9, 12) } return { "dsp_chart_type_country_code": { "apple": { "charts_daily": { **markets } } } } dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get(f"/api/charts/summary/", headers=auth, params=params) assert response.status == status assert dsp_send_mock.call_count == call_count[0] if status == HTTPStatus.OK: response = await response.json() assert response == expected_result @pytest.mark.parametrize( "params,status,expected_result,call_count", ( ({"isrc": "AAA123456789"}, HTTPStatus.BAD_REQUEST, None, 0), ( {"isrc": "AAA123456789", "dsp": "spotify"}, HTTPStatus.OK, { "m10": [ { "currentPosition": None, "entryDate": None, "entryPosition": None, "latestDate": None, "latestPosition": None, "latestUpdateDate": "2021-12-02", "listType": "Regional", "peakDate": None, "peakPosition": None, "previousPosition": None, "timeWindow": "Daily", }, { "currentPosition": 60, "entryDate": "2020-02-02", "entryPosition": 10, "latestDate": "2021-12-07", "latestPosition": 10, "latestUpdateDate": "2021-12-03", "listType": "Regional", "peakDate": "2020-03-04", "peakPosition": 10, "previousPosition": 20, "timeWindow": "Weekly", }, { "currentPosition": None, "entryDate": None, "entryPosition": None, "latestDate": None, "latestPosition": None, "latestUpdateDate": "2021-12-02", "listType": "Viral", "peakDate": None, "peakPosition": None, "previousPosition": None, "timeWindow": "Daily", }, { "currentPosition": None, "entryDate": None, "entryPosition": None, "latestDate": None, "latestPosition": None, "latestUpdateDate": "2021-12-03", "listType": "Viral", "peakDate": None, "peakPosition": None, "previousPosition": None, "timeWindow": "Weekly", }, ], "m11": [ { "currentPosition": None, "entryDate": None, "entryPosition": None, "latestDate": None, "latestPosition": None, "latestUpdateDate": "2021-12-03", "listType": "Regional", "peakDate": None, "peakPosition": None, "previousPosition": None, "timeWindow": "Daily", }, { "currentPosition": None, "entryDate": None, "entryPosition": None, "latestDate": None, "latestPosition": None, "latestUpdateDate": "2021-12-04", "listType": "Regional", "peakDate": None, "peakPosition": None, "previousPosition": None, "timeWindow": "Weekly", }, { "currentPosition": 61, "entryDate": "2020-02-02", "entryPosition": 11, "latestDate": "2021-12-06", "latestPosition": 11, "latestUpdateDate": "2021-12-03", "listType": "Viral", "peakDate": "2020-03-04", "peakPosition": 11, "previousPosition": 19, "timeWindow": "Daily", }, { "currentPosition": None, "entryDate": None, "entryPosition": None, "latestDate": None, "latestPosition": None, "latestUpdateDate": "2021-12-04", "listType": "Viral", "peakDate": None, "peakPosition": None, "previousPosition": None, "timeWindow": "Weekly", }, ], "m9": [ { "currentPosition": 59, "entryDate": "2020-02-09", "entryPosition": 9, "latestDate": "2021-12-06", "latestPosition": 9, "latestUpdateDate": "2021-12-01", "listType": "Regional", "peakDate": "2020-03-02", "peakPosition": 9, "previousPosition": 21, "timeWindow": "Daily", }, { "currentPosition": 59, "entryDate": "2020-02-10", "entryPosition": 14, "latestDate": "2021-12-07", "latestPosition": 13, "latestUpdateDate": "2021-12-02", "listType": "Regional", "peakDate": "2020-03-03", "peakPosition": 11, "previousPosition": 21, "timeWindow": "Weekly", }, { "currentPosition": 59, "entryDate": "2020-02-09", "entryPosition": 15, "latestDate": "2021-12-06", "latestPosition": 14, "latestUpdateDate": "2021-12-01", "listType": "Viral", "peakDate": "2020-03-02", "peakPosition": 12, "previousPosition": 21, "timeWindow": "Daily", }, { "currentPosition": 59, "entryDate": "2020-02-10", "entryPosition": 20, "latestDate": "2021-12-07", "latestPosition": 18, "latestUpdateDate": "2021-12-02", "listType": "Viral", "peakDate": "2020-03-03", "peakPosition": 14, "previousPosition": 21, "timeWindow": "Weekly", }, ], }, 4, ), ( {"isrc": "AAA123456789", "dsp": "apple", "type": "daily", "list_type": "regional"}, HTTPStatus.OK, { "m10": [ { "currentPosition": 60, "entryDate": "2020-02-01", "entryPosition": 10, "latestDate": "2021-12-06", "latestPosition": 10, "latestUpdateDate": "2021-12-02", "peakDate": "2020-03-03", "peakPosition": 10, "previousPosition": 20, }, ], "m11": [ { "currentPosition": 61, "entryDate": "2020-02-02", "entryPosition": 11, "latestDate": "2021-12-06", "latestPosition": 11, "latestUpdateDate": "2021-12-03", "peakDate": "2020-03-04", "peakPosition": 11, "previousPosition": 19, }, ], "m9": [ { "currentPosition": 59, "entryDate": "2020-02-09", "entryPosition": 9, "latestDate": "2021-12-06", "latestPosition": 9, "latestUpdateDate": "2021-12-01", "peakDate": "2020-03-02", "peakPosition": 9, "previousPosition": 21, }, ], }, 4, ), ), ) async def test_get_tracks_summary( params: dict, status: int, expected_result: list, call_count: int, mocker, auth, client ): isrc = params.pop("isrc") async def dsp_send(url: str, *args, **kwargs): is_apple = "apple" in url relative_url = url.replace("api/delphi/spotify/", "").replace("api/delphi/apple-music/", "").replace( "api/delphi/", "" ) if relative_url == "charts": chart_type_list, breakdown_list = ( kwargs["params"].get("type", ["charts"]), kwargs["params"].get("breakdown", ["daily"]) ) return { "items": [ { "chart_id": f"{chart_type}_{breakdown}_m{index}", "country_code": f"m{index}", "type": chart_type, "breakdown": breakdown, } for chart_type, breakdown, index in product(chart_type_list, breakdown_list, list(range(9, 12))) ] } elif relative_url == "charts/data-health/status": return get_chart_date( market_mapping={f"m{index}": f"2021-12-0{index % 9 + 1}" for index in range(9, 12)}, dsp="apple" if is_apple else "spotify", breakdowns=ChartBreakdown.values(), types=ChartType.values(), ) elif relative_url == "tracks/charts": breakdown_list, chart_type_list = ( ["daily"] if is_apple else params.get("type", "daily,weekly").split(","), ["charts"] if is_apple else params.get("list_type", "regional,viral").split(","), ) return { "items": [ { "chart_meta": {"chart_id": f"{chart_type}_{breakdown}_m{index}"}, "metrics": { "date": f"2021-12-0{index % 9 + 1}", "position": 50 + index, "previous_position": 30 - index, "date_streams": 1600 + index, "is_entry": not (index % 4), } } for chart_type, breakdown, index in product(chart_type_list, breakdown_list, list(range(9, 12))) ] } elif relative_url == "tracks/charts/lifetime": breakdown_list, chart_type_list = ( ["daily"] if is_apple else params.get("type", "daily,weekly").split(","), ["charts"] if is_apple else params.get("list_type", "regional,viral").split(","), ) return { "items": [ { "chart_meta": { "chart_id": f"{chart_type}_{breakdown}_m{i}", "country_code": f"m{i}", "type": chart_type, "breakdown": breakdown, }, "public_meta": {"isrc": kwargs["params"]["isrc"][0]}, "lifetime_metrics": { "earliest_position_date": f"2020-02-0{(i % 10 + 1) if i > 9 else i}", "earliest_position": 6 * j + 5 * k + i, "min_position_date": f"2020-03-0{(i % 9 + 2) if i > 8 else (i + 1)}", "min_position": 3 * j + 2 * k + i, "latest_position": 5 * j + 4 * k + i, "latest_position_date": (date(2021, 12, 9) - timedelta(days=i // 3)).isoformat(), }, } for i in range(9, 12) for j, chart_type in enumerate( chart_type_list if i < 10 or is_apple else [list(ChartType.values())[i % 10]] ) for k, breakdown in enumerate( breakdown_list if i < 10 or is_apple else [list(ChartBreakdown.values())[i % 10]] ) ] } dsp_send_mock = mocker.patch.object(DspApiClient, "_send_request", side_effect=dsp_send) response = await client.get(f"/api/charts/tracks/{isrc}/summary/", headers=auth, params=params) assert response.status == status assert dsp_send_mock.call_count == call_count if status == HTTPStatus.OK: response = await response.json() response = {k: sorted(v, key=lambda i: (i.get("listType"), i.get("timeWindow"))) for k, v in response.items()} assert response == expected_result