import json import time from datetime import datetime, timezone import pytest from app.config import UPDATE_TIMEOUT from app.logger import log import app.src.tests.insights.playlist_freshness.insights_steps as insights from app.src.tests.insights.playlist_freshness.snowflake_steps import get_latest_cdc_event from app.src.tests.insights.tests_common import playlists_to_test, PLAYLIST_TYPES """ https://theorchard.atlassian.net/browse/IN-15802 Check data from Insights api corresponds CDC event in database: 1. Get data from SPOTIFY_PRIORITY_PLAYLIST_PROCESSING_LOG 2. Check api returns the same fresh data """ @pytest.mark.parametrize("test_playlist", playlists_to_test, ids=lambda val: f"{val}") def test_playlist_freshness(graphql, snowflake, test_playlist, test_results): playlist_types = [ptype for ptype in PLAYLIST_TYPES if ptype != "USER_GENERATED"] log.info(f"------> Step1: find latest CDC event for playlist id {test_playlist}") cdc_event = get_latest_cdc_event(snowflake, test_playlist) if cdc_event is None: test_results["COMMENTS"].append('No CDC event found') finish_test(test_playlist, test_results) # retrieve latest update timestamp latest_cdc_timestamp = json.loads(cdc_event['UPDATE_SENT'])[0] log.info(f"Latest CDC update: {latest_cdc_timestamp}") # check how long ago update came and decrease UPDATE TIMEOUT latest_cdc_timestamp = latest_cdc_timestamp.replace(" Z", " +0000") update_time = datetime.strptime(latest_cdc_timestamp, "%a, %d %b %Y %H:%M:%S %z") now = datetime.now(timezone.utc) waited_time = (now - update_time).total_seconds() # Get ISRC to verify log.info("------> Step2: get ISRC from CDC event") track_objects = json.loads(cdc_event['TRACK_OBJECTS']) isrc_position = 0 # it gives us more chances to find playlist in top 500 placements isrc = track_objects[isrc_position]['isrc'] isrc_metadata = insights.get_insights_isrc_data(isrc, graphql) if isrc_metadata is None: test_results["COMMENTS"].append('No ISRC metadata found') finish_test(test_playlist, test_results, isrc=isrc) # Find artists artist_ids = insights.get_isrc_artist(isrc_metadata) if not artist_ids: test_results["COMMENTS"].append('No artists found') # Find products upcs = insights.get_isrc_product(isrc_metadata) if not upcs: test_results["COMMENTS"].append('No products found') log.info(f"ISRC: {isrc}") def api_equals_cdc_for(entity, _id): log.info(f"------> Step3: check {entity} {_id} has playlist {test_playlist}") placements = None if entity == 'track': placements = insights.get_insights_isrc_playlists(_id, graphql, playlist_types) elif entity == 'artist': placements = insights.get_insights_artist_playlists(_id, graphql, playlist_types) elif entity == 'product': placements = insights.get_insights_product_playlists(_id, graphql, playlist_types) if entity == 'track': playlist = insights.find_playlist_in_placements(test_playlist, placements) else: playlist = insights.find_playlist_in_placements(test_playlist, placements, isrc) if playlist is None: test_results["COMMENTS"].append(f"'{entity} {_id}' HAS NO playlist '{test_playlist}'") return None log.debug(playlist) if playlist['playlistType'] == 'PERSONALIZED': test_results["COMMENTS"].append(f"Playlist is PERSONALIZED for '{entity} {_id}'") return playlist['currentPosition'] is None return playlist['currentPosition'] == isrc_position + 1 if waited_time > UPDATE_TIMEOUT: log.info(f"DO VERIFICATION for {isrc}") if api_equals_cdc_for('track', isrc): test_results["ISRC"] = True log.info("Insights ISRC has updated position") for artist_id in artist_ids: test_results["ARTIST"] = api_equals_cdc_for('artist', artist_id) for upc in upcs: test_results["PRODUCT"] = api_equals_cdc_for('product', upc) else: while waited_time < UPDATE_TIMEOUT: log.info(f"WAIT AND DO VERIFICATION for {isrc}") if api_equals_cdc_for('track', isrc): test_results["ISRC"] = True log.info("Insights ISRC has updated position") for artist_id in artist_ids: test_results["ARTIST"] = api_equals_cdc_for('artist', artist_id) for upc in upcs: test_results["PRODUCT"] = api_equals_cdc_for('product', upc) break time.sleep(60) waited_time += 60 else: test_results["ISRC"] = False finish_test(test_playlist, test_results, waited_time, isrc, artist_ids, upcs) def finish_test(test_playlist, test_results, waited_time = 0, isrc = None, artist_ids = None, upcs = None): mins, secs = divmod(int(waited_time), 60) log.info("\n\n-------------RESULTS-------------\n" f"Playlist ID: {test_playlist}\n" f"Time: {mins} min {secs} sec\n\n" f"Song ISRC: {test_results['ISRC']} - {isrc}\n" f"Artist ID: {test_results['ARTIST']} - {artist_ids}\n" f"Product UPC: {test_results['PRODUCT']} - {upcs}\n" f"Comments: {test_results['COMMENTS']}" ) results = [test_results["ISRC"], test_results["ARTIST"], test_results["PRODUCT"]] assert all(results)