import logging import pandas import pytest from jsonbender import F as ApplyFunction from jsonbender import S as GetField from parameterized import parameterized from ui_automation_framework.utils import logger as cl import app.src.tests.decomission.api_init as api_init from app.src.tests.decomission.compare_helper import format_color, generate_provider, get_field_with_diff, get_field_with_diff_custom, get_field_with_diff_custom_date, get_field_with_diff_custom_date_or_zero, get_field_with_diff_delphi_not_empty, merge_arrays, transform_json API = api_init.APIOverride() log = cl.Logger(logging.DEBUG) apollo_to_delphi_keys_local = { "playlist_id": GetField("spotifyLink") >> ApplyFunction(lambda val: val.replace(":playlist:", "_")), "name": GetField("name"), "owner": GetField("owner") >> ApplyFunction(lambda val: 0 if val is None else ("spotify_" + val["accountId"])), "owner_name": GetField("owner") >> ApplyFunction(lambda val: 0 if val is None else val["accountName"]), "owner_category": GetField("owner") >> ApplyFunction(lambda val: 0 if val is None else val["categoryId"]), "track_count": GetField("trackCount"), "is_personalized": GetField("is_personalized"), "country_code": GetField("countryCode") >> ApplyFunction(lambda val: "global" if val == "_gl" else str(val).lower()), # streams "followers": GetField("subscribers"), "playlist_streams": GetField("marketStreams7Days"), "streams": GetField("streams", "local1Day") >> ApplyFunction(lambda val: 0 if val is None else val), "7_streams": GetField("streams", "local7Days") >> ApplyFunction(lambda val: 0 if val is None else val), # historical values "current": GetField("position"), "trend": GetField("positionChange") >> ApplyFunction(lambda val: 0 if val is None else val), "trend_date": GetField("positionChangeDate") >> ApplyFunction(lambda val: "" if val is None else str(val).split("T")[0]), "added": GetField("added") >> ApplyFunction(lambda val: "" if val is None else str(val).split("T")[0]), "earliest_added": GetField("earliestAdded") >> ApplyFunction(lambda val: "" if val is None else val.split("T")[0]), "album_name": "", "upc": "", } apollo_to_delphi_keys_ww = apollo_to_delphi_keys_local.copy() apollo_to_delphi_keys_ww.update({"streams": GetField("streams", "global1Day")}) apollo_to_delphi_keys_ww.update({"7_streams": GetField("streams", "global7Days")}) apollo_to_delphi_keys_ww.update({"playlist_streams": GetField("globalStreams7Days")}) delphi_to_delphi_keys_local = { # playlist object "playlist_id": GetField("playlist_id"), "name": GetField("playlist", "name"), "owner": GetField("playlist", "owner") >> ApplyFunction(lambda val: 0 if val is None else val["username"]), "owner_name": GetField("playlist", "owner") >> ApplyFunction(lambda val: 0 if val is None else val["display_name"]), "owner_category": GetField("playlist", "owner") >> ApplyFunction(lambda val: 0 if val is None or val["owner_category"] is None else int(val["owner_category"]["owner_category_id"])), "track_count": GetField("playlist", "num_tracks"), "is_personalized": GetField("playlist", "is_personalised"), "country_code": GetField("playlist", "country_code"), # streams (depending on market) "followers": GetField("playlist", "followers"), "playlist_streams": GetField("streaming_info", "us") >> ApplyFunction(lambda val: 0 if val is None or val["7"] is None else val["7"]["playlist"]), # better to give as 0 # ww for global "streams": GetField("streaming_info", "us") >> ApplyFunction(lambda val: 0 if val is None or val["1"] is None or val["1"]["isrc_in_playlist"] is None else val["1"]["isrc_in_playlist"]), # need to add conditions for using for any market "7_streams": GetField("streaming_info", "us") >> ApplyFunction(lambda val: 0 if val is None or val["7"] is None or val["7"]["isrc_in_playlist"] is None else val["7"]["isrc_in_playlist"]), # need to add conditions for using for any market # historical values "current": GetField("current"), "trend": GetField("trend_change_14_days") >> ApplyFunction(lambda val: 0 if val is None else val), "trend_date": GetField("last_date_change_14_days") >> ApplyFunction(lambda val: "" if val is None else str(val).split("T")[0]), "added": GetField("last_added_date_time") >> ApplyFunction(lambda val: "" if val is None else str(val).split("T")[0]), "earliest_added": GetField("earliest_position_date_time") >> ApplyFunction(lambda val: "" if val is None else val.split("T")[0]), "album_name": GetField("spotify_track", "album") >> ApplyFunction(lambda val: "" if val is None or val["name"] is None else val["name"]), "upc": GetField("spotify_track", "album") >> ApplyFunction(lambda val: "" if val is None or val["upc"] is None else val["upc"]), } delphi_to_delphi_keys_ww = delphi_to_delphi_keys_local.copy() delphi_to_delphi_keys_ww.update({"streams": GetField("streaming_info", "worldwide") >> ApplyFunction(lambda val: 0 if val is None or val["1"] is None else val["1"]["isrc_in_playlist"])}) delphi_to_delphi_keys_ww.update({"7_streams": GetField("streaming_info", "worldwide") >> ApplyFunction(lambda val: 0 if val is None or val["7"] is None else val["7"]["isrc_in_playlist"])}) delphi_to_delphi_keys_ww.update({"playlist_streams": GetField("streaming_info", "worldwide") >> ApplyFunction(lambda val: 0 if val is None or val["7"] is None else val["7"]["playlist"])}) apollo_offset = 0 apollo_limit = 2000 delphi_limit = 10000 apollo_portal_ww = "/gate-api/playlists/spotify/by-track/?isrc={0}&limit={1}&offset={2}&date={3}" delphi_ww = "/v3/public/track-positions/playlists?dsp=spotify&isrc={0}&limit={1}&include=playlists,streams_for_period,track_info" apollo_portal_local = "/gate-api/playlists/spotify/by-track/?isrc={0}&limit={1}&offset={2}&date={3}&market={4}" delphi_local = "/v3/public/track-positions/playlists?dsp=spotify&isrc={0}&limit={1}&include=playlists,streams_for_period,track_info" "&streams_country_code=worldwide&streams_country_code={2}" @parameterized.expand( [ # ('USRC12204647', 'au') # as it was # ("USSM12200612", "us"), # flowers ("USSM12209777", "ww"), ("USJI10800160", "us"), ("GBARL1300107", "ww"), ("GBARL1300522", "ww"), ("USSM12300722", "us"), ("USSM10804556", "us"), ("USSM12210918", "se"), ] ) @pytest.mark.skip() def test_current_playlists_spotify(isrc, market): spotify_latest_date = API.get_latest_date()["spotify"] apollo_response_all = [] if market == "ww": playlists_count = API.get_apollo_response(apollo_portal_ww.format(isrc, apollo_limit, apollo_offset, spotify_latest_date))["pagination"]["total"] for offset in range(0, playlists_count, apollo_limit): apollo_response = API.get_apollo_response(apollo_portal_ww.format(isrc, apollo_limit, offset, spotify_latest_date))["items"] apollo_response_all += apollo_response print(f"All data apollo response len: {len(apollo_response_all)}") delphi_response = API.get_delphi_response(delphi_ww.format(isrc, delphi_limit)) apollo_transformed_response = transform_json(apollo_to_delphi_keys_ww, apollo_response_all) delphi_transformed_response = transform_json(delphi_to_delphi_keys_ww, delphi_response["items"]) apollo_total = playlists_count else: playlists_count = API.get_apollo_response(apollo_portal_local.format(isrc, apollo_limit, apollo_offset, spotify_latest_date, market))["pagination"]["total"] for offset in range(0, playlists_count, apollo_limit): apollo_response = API.get_apollo_response(apollo_portal_local.format(isrc, apollo_limit, offset, spotify_latest_date, market))["items"] apollo_response_all += apollo_response print(f"All data apollo response len: {len(apollo_response_all)}") delphi_response = API.get_delphi_response(delphi_local.format(isrc, delphi_limit, market)) apollo_transformed_response = transform_json(apollo_to_delphi_keys_local, apollo_response_all) delphi_transformed_response = transform_json(delphi_to_delphi_keys_local, delphi_response["items"]) apollo_total = playlists_count delphi_total = delphi_response["count"] aggregated_matrix = merge_arrays(apollo_transformed_response, delphi_transformed_response, "playlist_id") index_for_all_fields = list(range(1, len(generate_provider(aggregated_matrix)) + 1)) subset_to_mark = range(3, len(aggregated_matrix) * 5, 5) pandas.DataFrame( { "provider": generate_provider(aggregated_matrix), "playlist_id": get_field_with_diff("playlist_id", aggregated_matrix), "name": get_field_with_diff("name", aggregated_matrix), "owner": get_field_with_diff("owner", aggregated_matrix), "owner_name": get_field_with_diff("owner_name", aggregated_matrix), "owner_category": get_field_with_diff("owner_category", aggregated_matrix), "track_count": get_field_with_diff("track_count", aggregated_matrix), "is_personalized": get_field_with_diff("is_personalized", aggregated_matrix), "country_code": get_field_with_diff("country_code", aggregated_matrix), # streams "followers": get_field_with_diff_custom("followers", aggregated_matrix, 2), "playlist_streams": get_field_with_diff("playlist_streams", aggregated_matrix), "streams": get_field_with_diff("streams", aggregated_matrix), "7_streams": get_field_with_diff("7_streams", aggregated_matrix), # historical values "current": get_field_with_diff("current", aggregated_matrix), "trend": get_field_with_diff("trend", aggregated_matrix), "trend_date": get_field_with_diff_custom_date_or_zero("trend_date", aggregated_matrix), "added": get_field_with_diff_custom_date("added", aggregated_matrix), "earliest_added": get_field_with_diff_custom_date("earliest_added", aggregated_matrix), # maybe calc based on total days "album_name": get_field_with_diff_delphi_not_empty("album_name", aggregated_matrix), "upc": get_field_with_diff_delphi_not_empty("upc", aggregated_matrix), }, index=index_for_all_fields, ).style.apply(format_color, subset=(subset_to_mark, slice(None))).to_excel(f"decomission/spotify_track_current_playlists/{isrc}_{market}_spotify_current_playlist_total_apollo_{apollo_total}_delphi_{delphi_total}.xlsx")