"""Contains all the release correction logic.""" from typing import Dict, List, NamedTuple import config from config import graphql_gateway from constants import queries from constants.constants import ( ARTIST_NAME, ARTIST_ROLE, ) from constants.role_mappings import ( LOCALIZATION_ROLE_MAP, PARTICIPANT_ROLE_MAP, TRACK_LOCALIZATION_ROLE_MAP, ) from ddex_ingester_common.constants.ddex_providers import SME, SOM_LIVRE_VENDOR_ID from ddex_ingester_common.constants.ownership import OWNERSHIP_TYPE_MAPPING from ddex_ingester_common.constants.roles import ( DDEX_ARTIST_ROLE_REQUIRED_GENRES) from ddex_ingester_common.constants.swb_deal_types import FOR_DISTRIBUTION from ddex_ingester_common.helpers.lambda_warning import LambdaWarning from ddex_ingester_common.helpers.list import have_common_elements from ddex_ingester_common.helpers.metadata import ( get_country_id, get_language_id, get_us_publishing_obligation, ) from ddex_ingester_common.logging import utils as logging_utils from ddex_ingester_common.models.s3.body import ( Body as S3Context) from ddex_ingester_common.models.s3.genre import Genre as S3Genre from ddex_ingester_common.models.s3.track import ( Track as S3Track) from ddex_ingester_common.models.state_machine.body import ( Body as StateMachineContext) from ddex_ingester_common.release_correction.release_correction_constants import ( # noqa RELEASE_CORRECTION_PRODUCT_ARTIST_FIELDS as RC_ARTIST_FIELDS, RELEASE_CORRECTION_PRODUCT_FIELDS, RELEASE_CORRECTION_TRACK_FIELDS, ) from ddex_ingester_common.release_correction.release_correction_diffs import ( ReleaseCorrectionDiffDetail) from ddex_ingester_common.release_correction.release_corrections_s3 import ( write_rc_json) from marshmallow.utils import get_value from utils import ( get_participations, get_title_localizations, is_track_role, ) logger = logging_utils.get_logger(config.app_logger) def diff_for_release_corrections( context: StateMachineContext, s3_data: S3Context, graphql_result: Dict, s3_release_corrections: Dict): """Check for items requiring release correction.""" logger.info('Checking for differences between Orchard and DDEX to add to release correction') # noqa # Check invalid fields before proceeding. rc_details = [] event_details = {'bucket': context.bucket, 'key': context.key} check_for_unsupported_updates( context, s3_data, graphql_result, rc_details, s3_release_corrections, event_details) fields = { 'productName': ReleaseCorrectionValuePair( old=graphql_result.get('productName'), new=s3_data.product.product_name), 'deliveredVersion': ReleaseCorrectionValuePair( old=graphql_result.get('deliveredVersion'), new=s3_data.product.product_version), 'genreId': ReleaseCorrectionValuePair( old=graphql_result.get('genreId'), new=context.product.genre_id), 'subgenreId': ReleaseCorrectionValuePair( old=graphql_result.get('subgenreId'), new=context.product.subgenre_id), 'cLine': ReleaseCorrectionValuePair( old=graphql_result.get('cLine'), new=s3_data.product.c_line), 'imprint': ReleaseCorrectionValuePair( old=graphql_result.get('imprint'), new=s3_data.product.imprint), } graphql_participations = graphql_result.get('labelParticipations', []) s3_participations = get_participations( s3_data.product.display_artists, s3_data.label_participants, s3_data.product.genres, context, True ) formatted_s3_participations = format_s3_participations(s3_participations) formatted_gql_participations =\ format_graphql_participations(graphql_participations) participation_diffs = compare_list_of_dicts( formatted_s3_participations, formatted_gql_participations) if participation_diffs: logger.info(f'Participation diffs found: {participation_diffs}') mapped_roles = set() for p in formatted_s3_participations: new_value = { ARTIST_ROLE: p[ARTIST_ROLE], ARTIST_NAME: p[ARTIST_NAME] } rc_details.append(create_release_correction_detail( new_value[ARTIST_ROLE], None, [new_value] )) mapped_roles.add(p[ARTIST_ROLE]) invalid_roles = get_genre_invalid_roles(s3_data.product.genres) # Clear roles that have no artists in the DDEX for role in RC_ARTIST_FIELDS: if role not in mapped_roles and role not in invalid_roles: mapped_roles.add(role) rc_details.append(create_release_correction_detail( role, None, [None])) for key in fields.keys(): if fields[key].new and fields[key].old != fields[key].new: rc_details.append(create_release_correction_detail( key, fields[key].old, check_for_format_requirements(fields, key))) if rc_details: s3_release_corrections['changes'].extend(rc_details) write_rc_json(event_details, s3_release_corrections) def create_release_correction_detail( field_name: str, old_value: any, new_value: any) -> ReleaseCorrectionDiffDetail: """Create ReleaseCorrectionDiffDetail using given values.""" logger.info(f'Creating Product Release Correction Detail' f' for field: {field_name}' f' with old value: {old_value}' f' and new value: {new_value}') rc_product_field = RELEASE_CORRECTION_PRODUCT_FIELDS[field_name] return ReleaseCorrectionDiffDetail( field_name=field_name, isrc=None, old=old_value, new=new_value, db_table_name=rc_product_field.db_table_name, db_field_name=rc_product_field.db_field_name, email_customer=rc_product_field.email_customer, accept_update=rc_product_field.accept_update) def create_track_release_correction_detail( field_name: str, isrc: str, old_value: any, new_value: any) -> ReleaseCorrectionDiffDetail: """Create ReleaseCorrectionDiffDetail for a track using given values.""" logger.info(f'Creating Track Release Correction Detail' f' for field: {field_name}' f' with old value: {old_value}' f' and new value: {new_value}') rc_product_field = RELEASE_CORRECTION_TRACK_FIELDS[field_name] return ReleaseCorrectionDiffDetail( field_name=field_name, isrc=isrc, old=old_value, new=new_value, db_table_name=rc_product_field.db_table_name, db_field_name=rc_product_field.db_field_name, email_customer=rc_product_field.email_customer, accept_update=rc_product_field.accept_update) def format_graphql_product_localizations(localizations: List[Dict]) -> List: """Format GraphQL product localizations for comparison.""" formatted_localizations = [] for local in localizations: formatted = { 'languageId': local.get('iTunesLanguage').get('id'), 'productName': local.get('productName'), } delivered_version = local.get('deliveredVersion') if delivered_version: formatted['deliveredVersion'] = delivered_version formatted_localizations.append(formatted) return formatted_localizations def format_gql_product_artist_localizations(localizations: List[Dict]) -> List: """Format GraphQL product artist localizations for comparison.""" formatted_localizations = [] for product_localization in localizations: for artist in product_localization['artists']: formatted_localizations.append({ 'languageId': product_localization['iTunesLanguage']['id'], 'artistName': artist['artistName'], 'artistType': artist['artistType'], }) return formatted_localizations def format_s3_product_artist_localizations(localizations: List[Dict]) -> List: """Format S3 product artist localizations for comparison.""" formatted_localizations = [] for artist in localizations: for localization in artist['localizations']: formatted_localizations.append({ 'languageId': localization['languageId'], 'artistName': localization['name'], 'artistType': LOCALIZATION_ROLE_MAP[artist['role']], }) return formatted_localizations def format_s3_track_localizations(track: S3Track) -> List: """Format S3 track title localizations for comparison.""" formatted_localizations = [] for localized_title in track.localized_titles or []: formatted_localizations.append({ 'languageId': get_language_id(localized_title.language_code), 'trackName': localized_title.title, 'version': localized_title.version or '', }) return formatted_localizations def format_gql_track_localizations(track_localizations: List) -> List: """Format GraphQL track title localizations for comparison.""" formatted_localizations = [] for localization in track_localizations or []: formatted_localizations.append({ 'languageId': localization['iTunesLanguage']['id'], 'trackName': localization['trackName'], 'version': localization['version'], }) return formatted_localizations def format_s3_track_artist_localizations( track: S3Track, genres: List[S3Genre], skip_genre_check: bool = False) -> List: """Format S3 track artist localizations for comparison.""" formatted_localizations = [] for artist in track.display_artists or []: for role in artist.roles or []: mapped_role = TRACK_LOCALIZATION_ROLE_MAP.get(role) track_role = is_track_role( role, genres, skip_genre_check=skip_genre_check) if track_role and mapped_role: for localized_name in artist.localized_names or []: formatted_localizations.append({ 'languageId': get_language_id( localized_name.language_code), 'artistName': localized_name.name, 'artistType': mapped_role, }) return formatted_localizations def format_gql_track_artist_localizations(track_localizations: List) -> List: """Format GraphQL track artist localizations for comparison.""" formatted_localizations = [] for localization in track_localizations or []: for artist in localization.get('artists') or []: formatted_localizations.append({ 'languageId': localization['iTunesLanguage']['id'], 'artistName': artist['artistName'], 'artistType': artist['artistType'], }) return formatted_localizations def format_s3_participations(participations: List[Dict]) -> List: """Format S3 participations list for comparison.""" formatted_s3 = [] for p in participations: part = { ARTIST_NAME: p.get(ARTIST_NAME), ARTIST_ROLE: PARTICIPANT_ROLE_MAP[p.get('role')] } formatted_s3.append(part) return formatted_s3 def format_graphql_participations(participations: List[Dict]) -> List: """Format GraphQL participations list for comparison.""" formatted_gql = [] for p in participations: part = { ARTIST_ROLE: p['role'], ARTIST_NAME: p['labelParticipant']['name'] } formatted_gql.append(part) return formatted_gql def check_for_format_requirements(fields: Dict, key: str) -> any: """Check if keys require special value formatting.""" if key == 'subgenreId': return [fields[key].new] return fields[key].new def compare_list_of_dicts( s3_participations: List[Dict], graphql_participations: List[Dict]) -> List: """Compare product participations for release correction.""" list_a_differences = [ item for item in s3_participations if item not in graphql_participations ] list_b_differences = [ item for item in graphql_participations if item not in s3_participations ] return [*list_a_differences, *list_b_differences] def check_for_unsupported_updates( context: StateMachineContext, s3_data: S3Context, graphql_result: Dict, rc_details: List[Dict], s3_release_corrections: Dict, event: Dict): """Check if unsupported fields are updated.""" logger.info('Checking for unsupported updates.') # This function also checks for unsupported updates on track fields # This is done to avoid raising an exception after this lambda # created release correction items which would have to be reverted error_message = '' if context.product.vendor_id != SOM_LIVRE_VENDOR_ID: error_message += check_for_invalid_product_code_update( s3_data, graphql_result, rc_details, s3_release_corrections) error_message += check_for_invalid_isrc_update( s3_data, graphql_result, rc_details, s3_release_corrections) if context.product.not_for_distribution == FOR_DISTRIBUTION: error_message += check_for_invalid_product_localization_update( context, s3_data, graphql_result, rc_details, s3_release_corrections ) for s3_track in s3_data.tracks or []: gql_track = next( (t for t in graphql_result.get('tracks') or [] if t['isrc'] == s3_track.isrc), {} ) error_message += check_for_track_number_update( s3_track, gql_track, rc_details, s3_release_corrections ) if context.product.not_for_distribution == FOR_DISTRIBUTION: if context.product.vendor_id != SOM_LIVRE_VENDOR_ID: error_message += check_for_invalid_ownership_rights_update( s3_track, gql_track, rc_details, s3_release_corrections, ) error_message += check_for_invalid_recording_country_update( s3_track, gql_track, rc_details, s3_release_corrections, ) error_message += check_for_invalid_original_rights_holder_country_update( # noqa s3_track, gql_track, rc_details, s3_release_corrections, ) error_message += check_for_invalid_publisher_update( context, s3_track, gql_track, ) error_message += check_for_invalid_us_publishing_obligation_update( s3_track, gql_track, rc_details, s3_release_corrections, ) # AWAL always maps every role regardless of genre skip_genre_check = context.ddex_provider != SME error_message += check_for_invalid_track_localization_update( s3_track, gql_track, rc_details, s3_release_corrections, s3_data.product.genres, skip_genre_check=skip_genre_check, ) if error_message: # Remove last \n error_message = error_message[:-1] write_rc_json(event, s3_release_corrections) raise ReleaseCorrectionUpdateException(error_message) def check_for_invalid_product_code_update( s3_data: S3Context, graphql_result: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update to productCode.""" product_code = graphql_result.get('productCode') if product_code and product_code != s3_data.product.catalog_number: field_name = 'productCode' rc_details.append(create_release_correction_detail( field_name, product_code, s3_data.product.catalog_number)) s3_release_corrections['changes'].extend(rc_details) return ( f'Product Code update attempt: {s3_data.product.catalog_number}' f' with existing Product Code: {product_code}\n') return '' def check_for_invalid_isrc_update( s3_data: S3Context, graphql_result: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on track ISRC.""" # OR to handle None and empty list as the set takes priority over both graphql_tracks = graphql_result.get('tracks') or set() if graphql_tracks: graphql_tracks = set(track['isrc'] for track in graphql_tracks) s3_tracks = s3_data.tracks or set() if s3_tracks: s3_tracks = set(track.isrc for track in s3_tracks) if graphql_tracks != s3_tracks: # Convert to list as sets are not serializable graphql_tracks = list(graphql_tracks) s3_tracks = list(s3_tracks) field_name = 'isrc' rc_details.append(create_track_release_correction_detail( field_name, None, graphql_tracks, s3_tracks)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) return ( f'Track ISRC update attempt. DDEX tracks {s3_tracks}' f" do not match The Orchard's tracks {graphql_tracks}\n") return '' def check_for_track_number_update( s3_track: S3Track, gql_track: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on track number.""" current_number = gql_track.get('trackNumber') if current_number and current_number != s3_track.sequence_number: field_name = 'trackNumber' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, current_number, s3_track.sequence_number)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) return ( f'Track number update attempt on track {s3_track.isrc}. ' f'DDEX track number ' f"{s3_track.sequence_number} does not match The Orchard's " f'track number {current_number}\n') return '' def check_for_invalid_ownership_rights_update( s3_track: S3Track, gql_track: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on ownership rights.""" current_ownership_rights = gql_track.get('ownershipRights') new_ownership_rights = OWNERSHIP_TYPE_MAPPING.get( s3_track.ownership_rights) if current_ownership_rights \ and new_ownership_rights != current_ownership_rights: field_name = 'ownershipRights' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, current_ownership_rights, new_ownership_rights)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) return ( f'Ownership rights update attempt on track {s3_track.isrc}. ' f'DDEX ownership rights ' f"{new_ownership_rights} does not match The Orchard's " f'ownership rights {current_ownership_rights}\n') return '' def check_for_invalid_recording_country_update( s3_track: S3Track, gql_track: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on recording country.""" current_country_id = gql_track.get('recordingCountryId') or 0 new_country_id = get_country_id(s3_track.recording_country_code) or 0 # noqa None is not accepted for country ID by GraphQL if new_country_id != current_country_id: field_name = 'recordingCountry' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, current_country_id, new_country_id)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) return ( f'Recording country update attempt on track {s3_track.isrc}. ' f'DDEX recording country id ' f"{new_country_id} does not match The Orchard's " f'recording country id {current_country_id}\n') return '' def check_for_invalid_original_rights_holder_country_update( s3_track: S3Track, gql_track: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on original rights holder country.""" current_country_id = gql_track.get('originalRightsHolderCountryId') or 0 new_country_id = get_country_id(s3_track.copyright_owner_country) or 0 # noqa None is not accepted for country ID by GraphQL if new_country_id != current_country_id: field_name = 'originalRightsHolderCountry' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, current_country_id, new_country_id)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) return ( f'Original rights holder country update attempt ' f'on track {s3_track.isrc}. ' f'DDEX original rights holder country id {new_country_id} ' f"does not match The Orchard's " f'original rights holder country id {current_country_id}\n') return '' def check_for_invalid_publisher_update( context: StateMachineContext, s3_track: S3Track, gql_track: Dict) -> str: """Check for invalid update on publishers.""" current_publisher_names = get_value( gql_track, 'publishing.publisherNames') or [] new_publisher_names = s3_track.publishers or [] if type(context.warnings) is not list: context.warnings = [] error_message = '' if set(new_publisher_names) != set(current_publisher_names): field_name = 'publisherNames' # Write details context.warnings error_message = { 'isrc': s3_track.isrc, 'current_values': current_publisher_names, 'new_values': new_publisher_names } logger.info(f'Found publisher difference for track {s3_track.isrc}' f'Current publisher values: {current_publisher_names}' f'Update attempt publisher values: {new_publisher_names}') context.warnings.append( LambdaWarning( field_name, 'ReleaseCorrectionUpdateException', error_message )._asdict() ) return '' def check_for_invalid_us_publishing_obligation_update( s3_track: S3Track, gql_track: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on US publishing obligation.""" current_publishing_obligation = get_value( gql_track, 'publishing.usPublishingObligation') or '' new_publishing_obligation = get_us_publishing_obligation( s3_track.us_publishing_obligation) or '' if new_publishing_obligation != current_publishing_obligation: field_name = 'usPublishingObligation' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, current_publishing_obligation, new_publishing_obligation)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) return ( f'US publishing obligation update attempt ' f'on track {s3_track.isrc}. ' f'DDEX US publishing obligation {new_publishing_obligation} ' f"does not match The Orchard's " f'US publishing obligation {current_publishing_obligation}\n') return '' def check_for_invalid_product_localization_update( context: StateMachineContext, s3_data: S3Context, graphql_result: Dict, rc_details: List[Dict], s3_release_corrections: Dict) -> str: """Check for invalid update on product localization.""" message = '' s3_product_localizations = get_title_localizations(s3_data.product) gql_product_localizations = format_graphql_product_localizations( graphql_result.get('productLocalizations') ) product_localization_diffs = compare_list_of_dicts( s3_product_localizations, gql_product_localizations) if product_localization_diffs: field_name = 'productLocalization' rc_details.append(create_release_correction_detail( field_name, gql_product_localizations, s3_product_localizations)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) message += ( f'Product localization update attempt. DDEX product localizations ' f'{s3_product_localizations} ' f"do not match The Orchard's product localizations " f'{gql_product_localizations}\n') s3_participations = get_participations( s3_data.product.display_artists, s3_data.label_participants, s3_data.product.genres, context ) s3_artist_localizations = format_s3_product_artist_localizations( s3_participations ) gql_artist_localizations = format_gql_product_artist_localizations( graphql_result.get('productLocalizations') ) artist_localization_diffs = compare_list_of_dicts( s3_artist_localizations, gql_artist_localizations) if artist_localization_diffs: field_name = 'productArtistLocalization' rc_details.append(create_release_correction_detail( field_name, gql_artist_localizations, s3_artist_localizations)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) message += ( f'Product artist localization update attempt. ' f'DDEX product artist localizations ' f'{s3_artist_localizations} ' f" do not match The Orchard's product artist localizations " f'{gql_artist_localizations}\n') return message def check_for_invalid_track_localization_update( s3_track: S3Track, gql_track: Dict, rc_details: List[Dict], s3_release_corrections: Dict, genres: List[S3Genre], skip_genre_check: bool = False) -> str: """Check for invalid update on track localization.""" message = '' s3_track_localizations = format_s3_track_localizations(s3_track) gql_track_localizations = format_gql_track_localizations( gql_track.get('localizations') ) track_localization_diffs = compare_list_of_dicts( s3_track_localizations, gql_track_localizations) if track_localization_diffs: field_name = 'trackLocalization' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, gql_track_localizations, s3_track_localizations)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) message += ( f'Track localization update attempt on track {s3_track.isrc}. ' f'DDEX track localizations ' f'{s3_track_localizations} ' f" do not match The Orchard's track localizations " f'{gql_track_localizations}\n') s3_artist_localizations = format_s3_track_artist_localizations( s3_track, genres, skip_genre_check=skip_genre_check ) gql_artist_localizations = format_gql_track_artist_localizations( gql_track.get('localizations') ) artist_localization_diffs = compare_list_of_dicts( s3_artist_localizations, gql_artist_localizations) if artist_localization_diffs: field_name = 'trackArtistLocalization' rc_details.append(create_track_release_correction_detail( field_name, s3_track.isrc, gql_artist_localizations, s3_artist_localizations)) # Write details to json before we raise exception. s3_release_corrections['changes'].extend(rc_details) message += ( f'Track artist localization update attempt ' f'on track {s3_track.isrc}. ' f'DDEX track artist localizations ' f'{s3_artist_localizations} ' f" do not match The Orchard's track artist localizations " f'{gql_artist_localizations}\n') return message def get_genre_invalid_roles(genres: List[S3Genre]) -> set: """Get roles that do not apply to the product genres.""" roles = set() for role in DDEX_ARTIST_ROLE_REQUIRED_GENRES: required_genres = DDEX_ARTIST_ROLE_REQUIRED_GENRES[role] genre_names = [genre.genre for genre in genres] # Applies if any product genre matches one of the required genres if not have_common_elements(genre_names, required_genres): roles.add(role.lower()) return roles def update_product_nfd(context: StateMachineContext): """Update Product NFD status.""" payload = { 'data': { 'productId': context.product.product_id, 'notForDistribution': context.product.not_for_distribution, } } logger.info(f'Running update product for NFD with payload:\n{payload}') graphql_gateway.execute(queries.UPDATE_PRODUCT, payload) class ReleaseCorrectionUpdateException(Exception): """Release correction update exception.""" class ReleaseCorrectionValuePair(NamedTuple): """Contains the current and incoming values for a DDEX field.""" new: any old: any