import os from concurrent.futures import ThreadPoolExecutor, as_completed from app.logger import log from app.data import load_ids_csv from .insights_steps import fetch_playlist_tracklist PRIORITY_PLAYLISTS_ONLY = os.environ.get('PRIORITY_PLAYLISTS_ONLY', '') """ https://theorchard.atlassian.net/browse/IN-16866 Check playlist's tracks completeness: Compare tracks count between playlist list page → https://insights.theorchard.com/playlists and particular playlist page → e.g. https://insights.theorchard.com/playlist/37i9dQZF1DXcBWIGoYBM5M/ """ def test_playlist_completeness(graphql): # Load priority filter lazily priority_playlists = ( set(load_ids_csv('playlists_priority.csv')) if PRIORITY_PLAYLISTS_ONLY else None ) log.info(f"------> Step1: Get all playlists ids") playlists_ids = graphql.fetch_data('PlaylistIdsForNavigation')['playlistIds'] log.debug(f" There are total {len(playlists_ids)} playlists in the system.") log.info(f"------> Step2: Get ALL playlists metadata") playlists_metadata = graphql.fetch_data( 'PlaylistsMetadataListPaginated', {"inputs": playlists_ids, "limit": len(playlists_ids), "offset": 0}) # Saving results for reporting results_data = [] result = True # Prepare tasks for parallel execution tasks = [] for i, playlist_metadata in enumerate(playlists_metadata['playlistsMetadataListPaginated']['items']): playlist_id = playlist_metadata['playlistId'] playlist_store_id = playlist_metadata['source']['storeId'] track_count_from_metadata = playlist_metadata['playlistTrackCount'] if priority_playlists is not None and playlist_id not in priority_playlists: log.info(f"Skipping non-priority playlist: {playlist_id}") continue tasks.append((i, playlist_id, playlist_store_id, track_count_from_metadata)) log.info(f"------> Step3: Fetching {len(tasks)} playlists in parallel") # Execute requests in parallel with max 10 workers with ThreadPoolExecutor(max_workers=10) as executor: future_to_task = { executor.submit(fetch_playlist_tracklist, graphql, task[1], task[2]): task for task in tasks } for future in as_completed(future_to_task): i, playlist_id, playlist_store_id, track_count_from_metadata = future_to_task[future] try: playlist_tracklist = future.result() playlist_data = playlist_tracklist['playlist'] if playlist_data is None: log.error(f"Playlist id: {playlist_id} returned null for 'playlist'. Skipping.") results_data.append({ "assertion": False, "playlist_id": playlist_id, "track_count_from_metadata": track_count_from_metadata, "track_count_from_tracklist": None }) result = False continue track_count_from_tracklist = playlist_data['playlistPlacements']['totalCount'] assertion = track_count_from_metadata == track_count_from_tracklist results_data.append({ "assertion": assertion, "playlist_id": playlist_id, "track_count_from_metadata": track_count_from_metadata, "track_count_from_tracklist": track_count_from_tracklist }) if not assertion: log.error(f"Playlist id: {playlist_id} has different track counts: " f"{track_count_from_metadata} in metadata and {track_count_from_tracklist} in tracklist") result = False except Exception as e: log.error(f"Error fetching playlist {playlist_id}: {e}") result = False log.info("\n-------------RESULTS-------------\n") for index, data in enumerate(results_data): log.info(f"( {index + 1} ) : " f"PLAYLIST_ID: {data['playlist_id']} |" f"METADATA: {data['track_count_from_metadata']} | " f"TRACKLIST: {data['track_count_from_tracklist']} | " f"{data['assertion']}" ) assert result, f"Some playlists have different track counts in metadata and tracklist"