"""Approve Releases in Bulk via GraphQL.""" import argparse import numpy as np from connectors.logging import logger as log from constants import fields from constants.role_mapping_keys import ( BEATROOT_ROLE_MAPPING_KEY, WMG_STUDIO_ROLE_MAPPING_KEY ) # GraphQL from logic import graphql_logic from utils import graphql_utils # Role Mappings -- ADD NEW ROLE MAPPINGS HERE from constants.wmg_studio_role_mapping import WMG_STUDIO_ROLE_MAPPING from constants.beatroot_mapping import BEATROOT_ROLE_MAPPING import config def debug_config(): return config.VENDOR_ID \ and config.ISRC \ and config.SUBACCOUNT_ID \ and config.ARTIST_NAME \ and config.ARTIST_ROLE def get_file_fields(filename: str) -> list: """Get fields from a file. Args: filename (str): The filename to process. Returns: list: A list of dictionaries containing the fields. """ # Read an xlsx file if filename.endswith('.xlsx'): import pandas as pd # Read file df = pd.read_excel(filename) # Strip whitespace from all string fields df = df.apply( lambda col: col.map( lambda x: x.strip() if isinstance(x, str) else x)) # Blank spaces create `np.nan` values, replace them with None df = df.replace([np.nan], [None], regex=False) return_list = df.to_dict(orient="records") # Read a csv file elif filename.endswith('.csv'): import csv return_list = [] # Read file with open(filename, 'r') as file: reader = csv.DictReader(file) # Strip whitespace from all string fields for row in reader: return_list.append({k: v.strip() if isinstance(v, str) else v for k, v in row.items()}) return_list = list(reader) else: raise Exception('Invalid file type. Must be .csv or .xlsx.') return return_list def main(): """Attach Label Participants to Tracks via GraphQL.""" # Parse command-line arguments parser = argparse.ArgumentParser( description='Attach Label Participants to Tracks via GraphQL.') parser.add_argument( 'filename', type=str, help='The filename to process') # noqa args = parser.parse_args() # Message user msg = f'Using {config.ROLE_MAPPING} role mapping.' log.info(msg) # Check if the file exists local_file = True if args.filename: try: with open(args.filename): pass except FileNotFoundError: local_file = False # if a testing config is passed. if debug_config(): # Fetch subaccount_id from config, or use non-D3 value of '0' subaccount_id = config.SUBACCOUNT_ID if config.SUBACCOUNT_ID else 0 # Create a dummy list of one value track_list = [(config.VENDOR_ID, config.ISRC, subaccount_id)] elif config.SNOWFLAKE_SOURCE_TABLE: # Connect Snowflake from utils import snowflake_utils as snow_db # Get isrc, name and wmg role from Snowflake track_list = snow_db.get_snowflake_fields( field_list=fields.FIELD_LIST, table=config.SNOWFLAKE_SOURCE_TABLE, distinct=True) elif args.filename and local_file: # Process CLI file log.info(f'Processing file: {args.filename}') track_list = get_file_fields(args.filename) # lowercase and snakecase the file field names track_list = [ {k.lower().replace(' ', '_'): v for k, v in track.items()} for track in track_list ] else: raise Exception('Must include SNOWFLAKE_SOURCE_TABLE in config.') # Make sure that the track list is not empty if not len(track_list): raise Exception('Track list fetch query produced no rows.') msg = f'Processing {len(track_list)} tracks.' log.info(msg) # Ensure artist names are always strings (some spreadsheets coerce int-like # artist names to numbers which later breaks string logic like .replace()) try: for track in track_list: if isinstance(track, dict) and fields.ARTIST_NAME in track and \ track[fields.ARTIST_NAME] not in (None, '') and \ not isinstance(track[fields.ARTIST_NAME], str): track[fields.ARTIST_NAME] = str(track[fields.ARTIST_NAME]) except Exception as e: log.error(f"Error casting artist names to string: {e}") raise # Ensure all int-like fields have no trailing decimals\ try: for track in track_list: for key, value in track.items(): if key in fields.INTEGER_LIKE_FIELDS: track[key] = \ int(value) if value not in (None, '') else None except Exception as e: log.error(f"Error removing decimals from integer-like fields: {e}") raise # Dedupe the track_list list of dicts on the following fields: # 'vendor_id', 'subaccount_id', 'isrc', 'artist_name', 'artist_role', # 'spotify_id', 'apple_music_id' log.info('Deduping track list.') unique_tracks = [] seen = set() count = 0 # Dedupe track list. try: for track in track_list: count += 1 # Create a unique key for each track using the distinct fields identifier = ( track.get(fields.VENDOR_ID), track.get(fields.SUBACCOUNT_ID), track.get(fields.ISRC), track.get(fields.ARTIST_NAME).replace("'", "''"), track.get(fields.ARTIST_ROLE), track.get(fields.SPOTIFY_ID), track.get(fields.APPLE_MUSIC_ID)) if identifier not in seen: seen.add(identifier) # Add the identifier to the set unique_tracks.append({ fields.VENDOR_ID: track.get(fields.VENDOR_ID), fields.SUBACCOUNT_ID: track.get(fields.SUBACCOUNT_ID), fields.ISRC: track.get(fields.ISRC), fields.ARTIST_NAME: track.get(fields.ARTIST_NAME).replace("'", "''"), fields.ARTIST_ROLE: track.get(fields.ARTIST_ROLE), fields.SPOTIFY_ID: track.get(fields.SPOTIFY_ID), fields.APPLE_MUSIC_ID: track.get(fields.APPLE_MUSIC_ID) }) # Append the track to the unique_tracks list except Exception as e: log.error(f"Error deduping track list: {e}") raise log.info(f"Removed {count - len(unique_tracks)} repeated items.") track_list = unique_tracks # Set the track_list to the unique_tracks list log.info(f"Track list now has {len(track_list)} unique items.") # TODO: Add a check to make sure that the track list does not contain two # or more artists with the same name, vendor_id, and subaccount_id, but # different unique store ids - This should probably be SQL or pandas. # Get all unique participants from track list participant_list = set() participant_list.update([( track[fields.ARTIST_NAME], track[fields.VENDOR_ID], track[fields.SUBACCOUNT_ID], track[fields.SPOTIFY_ID], track[fields.APPLE_MUSIC_ID] ) for track in track_list]) log.info(f'Found {len(participant_list)} unique label participants.') log.info(f'Checking for duplicate artist names with different spotify/apple music ids.') # noqa check_list = set() check_list.update([( track[fields.ARTIST_NAME], track[fields.VENDOR_ID], track[fields.SUBACCOUNT_ID] ) for track in track_list]) if len(check_list) != len(participant_list): raise Exception( 'Duplicate artist names with different spotify/apple music ids.') # Get or generate all participant uuids from the graph participant_list, error_list = \ graphql_logic.get_or_create_all_label_participants( participant_list, config.SLEEP_TIME/3) # Short-circuit to avoid setting participants for testing if not config.SET_TRACK_PARTICIPANTS: log.info('Skipping track participant assignment.') log.info("bulk-insert-label-participants completed.") return # -- Begin set track participants -- # # Get role mapping -- ADD NEW ROLE MAPPINGS HERE if config.ROLE_MAPPING == WMG_STUDIO_ROLE_MAPPING_KEY: role_mapping = WMG_STUDIO_ROLE_MAPPING elif config.ROLE_MAPPING == BEATROOT_ROLE_MAPPING_KEY: role_mapping = BEATROOT_ROLE_MAPPING else: raise Exception( '`ROLE_MAPPING` env var must be set, and must be valid.') log.info(f"Mapping roles using '{config.ROLE_MAPPING}'") # Ensure all incoming roles are mapped all_roles = set([track[fields.ARTIST_ROLE] for track in track_list]) missing_roles = all_roles - set(role_mapping.keys()) if missing_roles: raise Exception( f'Missing roles in role mapping: {missing_roles}') # Update Participants log.info('Organizing participants by ISRC.') # Pivot data by ISRC participant_by_isrc_dict = \ graphql_utils.transform_track_participants_isrc( track_list, participant_list, role_mapping) deduped_participants = {} # TODO: Check participant_by_isrc_dict for duple roles/name pairs per ISRC # Create a set of tuples from the participants list for isrc, participants in participant_by_isrc_dict.items(): deduped_participants[isrc] = [] # Create new participants list, without duplicates seen = set() for participant in participants: # Create a tuple from the participant participant_tuple = ( participant[fields.PARTICIPANT_NAME], participant[fields.PARTICIPANT_VENDOR_ID], participant[fields.PARTICIPANT_SUBACCOUNT_ID], participant[fields.PARTICIPANT_ROLE], participant[fields.PARTICIPANT_CATEGORY] ) # If the participant is not in the set, add it to the new list if participant_tuple not in seen: seen.add(participant_tuple) deduped_participants[isrc].append(participant) # Replace the original participant dict with the deduped one participant_by_isrc_dict = deduped_participants num_tracks = len(participant_by_isrc_dict.keys()) log.info(f'Setting track participants on {num_tracks} ISRC\'s.') # Attach label participants to tracks graphql_logic.set_all_label_participants_isrc( participant_by_isrc_dict, config.SLEEP_TIME) log.info("bulk-insert-label-participants completed.") if __name__ == '__main__': main()