"""Validation rules.""" from datetime import datetime import os import re from typing import NamedTuple from bulk_metadata_ingester_common.constants.genres import GENRE_SUBGENRE_MAP from bulk_metadata_ingester_common.constants.languages import LANGUAGE_CODE_MAP from bulk_metadata_ingester_common.constants.ownership import OWNERSHIP_TYPES from bulk_metadata_ingester_common.constants.roles import ( PERFORMER_MAIN_ROLES, PERFORMER_TYPES, ) from bulk_metadata_ingester_common.constants.territories import COUNTRY_NAMES from bulk_metadata_ingester_common.utils.error import graphql_execute from config import graphql_gateway from constants import rule_error_codes as err_codes from constants.constants import ( EXPLICIT_CONTENT_TYPES, VALID_FILE_NAME_EXTENSIONS, VALID_RELEASE_TYPES, ) from constants.data_sources import JSON_FULL from constants.languages import LANGUAGE_LIST from constants.pricing import ( MUSIC_ALBUM, MUSIC_TRACK, PRICE_MAPPING) from constants.queries import ( GET_PRODUCT_BY_UPC, GET_TERRITORIES, ) from constants.territory_standards import ORCH_1_2016 from ddex_ingester_common.constants.catalog_ingestion import ( REJECT, SUCCESS, ) MAX_NR_VOLUMES = 12 class RuleResult(NamedTuple): """A result from a rule.""" validation_rule_id: int response: str = SUCCESS message: str = None def validate_mandatory_fields(data: dict, logger: object) -> RuleResult: """Validate all mandatory fields contain values.""" results = [] all_releases = [] for k, release in data.items(): all_releases.append(release) all_tracks = [] for release in all_releases: for k, track in release.tracks.items(): all_tracks.append(track) # Check all mandatory fields are not null results.append(_check_not_null( [r.release_name for r in all_releases], err_codes.NULL_RELEASE_NAME)) results.append(_check_not_null( [r.metadata_language for r in all_releases], err_codes.NULL_RELEASE_METADATA_LANGUAGE)) # noqa: E501 results.append(_check_not_null( [r.project_artist for r in all_releases], err_codes.NULL_PROJECT_ARTIST)) # noqa: E501 results.append(_check_not_null( [r.artist_country for r in all_releases], err_codes.NULL_ARTIST_COUNTRY)) # noqa: E501 results.append(_check_not_null( [r.primary_artists for r in all_releases], err_codes.NULL_PRIMARY_RELEASE_ARTIST)) # noqa results.append(_check_not_null( [r.original_release_date for r in all_releases], err_codes.NULL_RELEASE_DATE)) # noqa: E501 results.append(_check_not_null( [r.sale_start_date for r in all_releases], err_codes.NULL_START_DATE)) results.append(_check_not_null( [r.preorder_preview for r in all_releases], err_codes.NULL_PREORDER_PREVIEW)) # noqa: E501 results.append(_check_not_null( [r.release_type for r in all_releases], err_codes.NULL_RELEASE_TYPE)) results.append(_check_not_null( [r.imprint for r in all_releases], err_codes.NULL_IMPRINT)) results.append(_check_not_null( [r.genre for r in all_releases], err_codes.NULL_GENRE)) results.append(_check_not_null( [r.subgenre for r in all_releases], err_codes.NULL_SUBGENRE)) results.append(_check_not_null( [r.c_line for r in all_releases], err_codes.NULL_C_LINE)) results.append(_check_not_null( [r.p_line for r in all_releases], err_codes.NULL_P_LINE)) results.append(_check_not_null( [r.project_code for r in all_releases], err_codes.NULL_PROJECT_CODE)) results.append(_check_not_null( [t.file_name for t in all_tracks], err_codes.NULL_FILE_NAME)) results.append(_check_not_null( [t.volume for t in all_tracks], err_codes.NULL_VOLUME)) results.append(_check_not_null( [t.track_no for t in all_tracks], err_codes.NULL_TRACK_NO)) results.append(_check_not_null( [t.track_name for t in all_tracks], err_codes.NULL_TRACK_NAME)) results.append(_check_not_null( [t.artist for t in all_tracks], err_codes.NULL_TRACK_ARTIST)) results.append(_check_not_null( [t.track_audio_language for t in all_tracks], err_codes.NULL_TRACK_AUDIO_LANGUAGE)) # noqa: E501 results.append(_check_not_null( [t.artist for t in all_tracks], err_codes.NULL_TRACK_ARTIST)) results.append(_check_not_null( [t.explicit for t in all_tracks], err_codes.NULL_EXPLICIT)) results.append(_check_not_null( [t.third_party_publisher for t in all_tracks], err_codes.NULL_THIRD_PARTY_PUBLISHER)) # noqa: E501 results.append(_check_not_null( [t.track_p_info for t in all_tracks], err_codes.NULL_TRACK_P_INFO)) results.append(_check_not_null( [t.ownership_for_this_sound_recording for t in all_tracks], err_codes.NULL_OWNERSHIP_SOUND_RECORDING)) # noqa: E501 results.append(_check_not_null( [t.country_of_recording for t in all_tracks], err_codes.NULL_COUNTRY_OF_RECORDING)) # noqa: E501 results.append(_check_not_null( [t.nationality_of_original_copyright_owner for t in all_tracks], err_codes.NULL_NATIONALITY_OF_ORIGINAL_COPYRIGHT_HOLDER)) results.append(_check_not_null( [t.songwriters for t in all_tracks], err_codes.NULL_SONGWRITERS)) results.append(_check_not_null( [t.publishers for t in all_tracks], err_codes.NULL_PUBLISHERS)) results.append(_check_not_null( [t.performer_1_type for t in all_tracks], err_codes.NULL_PERFORMER_TYPE)) # noqa: E501 results.append(_check_not_null( [t.performer_1_legal_name for t in all_tracks], err_codes.NULL_PERFORMER_LEGAL_NAME)) # noqa: E501 results.append(_check_not_null( [t.performer_1_main_role for t in all_tracks], err_codes.NULL_PERFORMER_MAIN_ROLE)) # noqa: E501 # Remove the nulls so we only have Reject values here. results = [result for result in results if result] or None return results def _check_not_null(data: dict, error_code: str) -> RuleResult: """Check if field is not null.""" for item in data: if item is None: return RuleResult( error_code, REJECT, f'Blank field found: {error_code}' ) def _validate_upc(upc: int) -> bool: """Validate a upc. Args: upc (int): A UPC to validate. Returns: bool: Whether or not the UPC is valid. """ # UCP must be 12 characters long if len(str(upc)) != 12: raise ValueError('{upc} does not contain 12 characters.') # Checksum is last digit checksum = str(upc)[-1] if _calculate_upc_checksum(upc) == int(checksum): return True return False def _calculate_upc_checksum(upc: int) -> bool: """Calculate the checksum on a upc. Args: upc (int): A UPC to check. Returns: int: The checksum of the upc. """ if len(str(upc)) == 12: upc = str(upc)[0:-1] if len(str(upc)) != 11: raise ValueError(f'UPC value of length {len(str(upc))} is invalid.') even_items = [] odd_items = [] # Loop through UPC without checksum and store values for pos, num in enumerate(str(upc)): if (pos + 1) % 2 == 0: even_items.append(int(num)) else: odd_items.append(int(num)) # Create the check for 6 odd values odd_check = sum(odd_items) * 3 # Create the check for 5 even values even_check = sum(even_items) # Sum checks round up to nearest ten sum_of_checks = even_check + odd_check diff_from_ten = 10 - (sum_of_checks % 10) return diff_from_ten def _is_valid_isrc(value: str) -> bool: """Check if ISRC is valid.""" value = str(value).strip().replace(chr(160), '').replace('-', '') match = re.match(r'^[A-Za-z]{2}[-]?[0-9A-Za-z]{3}[-]?[0-9]{2}[-]?[0-9]{5}$', value, re.I) # noqa: E501 return bool(match) def require_valid_track_pline(data: dict, logger: object) -> RuleResult: """Validate track p-line.""" all_releases = [] for k, release in data.items(): all_releases.append(release) all_tracks = list(all_releases[0].tracks.values()) for t in all_tracks: p_line = t.track_p_info if p_line and not re.match('[0-9]{4}[ ]+\\S+', p_line, flags=re.ASCII): return RuleResult( err_codes.INVALID_TRACK_PLINE, REJECT, f'Invalid P-Line on track ' f'{t.volume}-{t.track_no}') def require_valid_cline(data: dict, logger: object) -> RuleResult: """Validate product c-line.""" for k, release in data.items(): c_line = release.c_line if c_line and not re.match('[0-9]{4}[ ]+\\S+', c_line, flags=re.ASCII): return RuleResult( err_codes.INVALID_CLINE, REJECT, f'Invalid product C-Line: {c_line}' ) def require_valid_pline(data: dict, logger: object) -> RuleResult: """Validate product c-line.""" for k, release in data.items(): p_line = release.p_line if p_line and not re.match('[0-9]{4}[ ]+\\S+', p_line, flags=re.ASCII): return RuleResult( err_codes.INVALID_PLINE, REJECT, f'Invalid product P-Line: {p_line}' ) def validate_release_type(data: dict, logger: object) -> RuleResult: """Validate product release type.""" for k, release in data.items(): release_type = release.release_type if not release_type or release_type.upper() not in VALID_RELEASE_TYPES: return RuleResult( err_codes.INVALID_RELEASE_TYPE, REJECT, f'Invalid release type: {release_type}' ) def validate_artist_country(data: dict, logger: object) -> RuleResult: """Validate artist countries.""" graphql_result = graphql_execute( graphql_gateway, GET_TERRITORIES, {'data': ORCH_1_2016}, logger )['data']['territories'] territories = set() for t in graphql_result: territories.add(t) for k, release in data.items(): artist_country = release.artist_country if not artist_country or artist_country not in territories: return RuleResult( err_codes.INVALID_ARTIST_COUNTRY, REJECT, f'Invalid artist country: {artist_country}' ) def validate_release_upc(data: dict, logger: object) -> RuleResult: """Validate album pricing details.""" for k, release in data.items(): upc = release.upc vendor_id = release.vendor_id if upc and not _validate_upc(upc): return RuleResult( err_codes.INVALID_UPC_CHECKSUM, REJECT, f'Invalid upc; failed checksum: {upc}' ) graphql_result = graphql_execute( graphql_gateway, GET_PRODUCT_BY_UPC, {'data': upc}, logger )['data']['productByUpc'] if graphql_result: status = graphql_result.get('status') gql_vendor = graphql_result.get('vendorId') if status == 'in_content': # We do not deal with in_content products return RuleResult( err_codes.INVALID_UPC_IN_CONTENT, REJECT, f'Invalid upc; status is in_content: {upc}' ) else: # We do have a status and it's not in_content # The vendor_ids must match to accept this scenario. if gql_vendor != vendor_id: return RuleResult( err_codes.INVALID_UPC_VENDOR_MISMATCH, REJECT, f'Invalid upc; vendor id mismatch: {gql_vendor}' ) def validate_album_pricing(data: dict, logger: object) -> RuleResult: """Validate album pricing details.""" for k, release in data.items(): album_pricing = release.album_pricing if not album_pricing or album_pricing not in PRICE_MAPPING[MUSIC_ALBUM]: # noqa: E501 return RuleResult( err_codes.INVALID_ALBUM_PRICING, REJECT, f'Invalid price type: {album_pricing}' ) def validate_track_pricing(data: dict, logger: object) -> RuleResult: """Validate album pricing details.""" for k, release in data.items(): for k, track in release.tracks.items(): track_pricing = track.track_pricing if not track_pricing or track_pricing not in PRICE_MAPPING[MUSIC_TRACK]: # noqa: E501 return RuleResult( err_codes.INVALID_TRACK_PRICING, REJECT, f'Invalid price type: {track_pricing}' ) def validate_release_date(data: dict, logger: object) -> RuleResult: """Validate release date.""" for k, release in data.items(): release_date = release.original_release_date try: if release_date != datetime.strptime( release_date, '%Y-%m-%d' ).strftime('%Y-%m-%d'): return RuleResult( err_codes.INVALID_RELEASE_DATE, REJECT, f'Invalid release date: {release_date}' ) except ValueError: return RuleResult( err_codes.INVALID_RELEASE_DATE, REJECT, f'Invalid release date: {release_date}' ) def validate_sale_start_date(data: dict, logger: object) -> RuleResult: """Validate sale start date.""" for k, release in data.items(): sale_start_date = release.sale_start_date try: if sale_start_date != datetime.strptime( sale_start_date, '%Y-%m-%d' ).strftime('%Y-%m-%d'): return RuleResult( err_codes.INVALID_SALE_START_DATE, REJECT, f'Invalid release date: {sale_start_date}' ) except ValueError: return RuleResult( err_codes.INVALID_SALE_START_DATE, REJECT, f'Invalid release date: {sale_start_date}' ) def validate_release_meta_language(data: dict, logger: object) -> RuleResult: """Validate release metadata language.""" for k, release in data.items(): metadata_language = release.metadata_language if metadata_language not in LANGUAGE_LIST: return RuleResult( err_codes.INVALID_RELEASE_META_LANGUAGE, REJECT, f'Invalid metadata language: {metadata_language}' ) def validate_manufacturer_upc(data: dict, logger: object) -> RuleResult: """Validate Genre and Subgenre values.""" for k, release in data.items(): manufacturer_upc = release.manufacturer_upc if manufacturer_upc: if not isinstance(manufacturer_upc, int): return RuleResult( err_codes.INVALID_MANUFACTURER_UPC_FORMAT, REJECT, f'Invalid manufacturer_upc; not int: {manufacturer_upc}' ) length = len(str(manufacturer_upc)) if not (length >= 12 and length <= 14): return RuleResult( err_codes.INVALID_MANUFACTURER_UPC_LENGTH, REJECT, f'Invalid manufacturer_upc length: {length}' ) def validate_gaps_track_volume_number(data: dict, logger: object) -> RuleResult: # noqa: E501 """Validate there are no gaps in track or volume numbers.""" # Assume max of X volumes volume_and_track_list = [[] for i in range(MAX_NR_VOLUMES)] for key, release in data.items(): for track_key, track in release.tracks.items(): volume = track.volume track_no = track.track_no volume_and_track_list[volume - 1].append(track_no) for volume_nr, volume_data in enumerate(volume_and_track_list): if not volume_data and len(volume_and_track_list) > volume_nr + 1 and volume_and_track_list[volume_nr + 1]: # noqa: E501 return RuleResult( err_codes.MISSING_TRACK_OR_VOLUME_NUMBER, REJECT, f'Missing volume number {volume_nr + 1}' ) sorted_volume = sorted(volume_data) expected_track_no = 1 for track_no in sorted_volume: if expected_track_no != track_no: return RuleResult( err_codes.MISSING_TRACK_OR_VOLUME_NUMBER, REJECT, f'Missing track number {expected_track_no} on volume {volume_nr + 1}' # noqa: E501 ) expected_track_no += 1 def validate_genre_subgenre(data: dict, logger: object) -> RuleResult: """Validate Genre and Subgenre values.""" for k, release in data.items(): genre = release.genre subgenre = release.subgenre if genre not in GENRE_SUBGENRE_MAP: return RuleResult( err_codes.INVALID_GENRE, REJECT, f'Invalid genre: {genre}' ) if subgenre not in GENRE_SUBGENRE_MAP[genre]: return RuleResult( err_codes.INVALID_SUBGENRE, REJECT, f'Invalid genre and subgenre: {genre}:{subgenre}' ) def validate_isrc(data: dict, logger: object) -> RuleResult: """Validate ISRC.""" for k, release in data.items(): known_isrcs = [] for k, track in release.tracks.items(): isrc = track.isrc if not isrc: continue # Check for dupes at product level if isrc not in known_isrcs: known_isrcs.append(isrc) else: return RuleResult( err_codes.INVALID_ISRC_DUPLICATE, REJECT, f'Invalid ISRC; duplicate ISRC within product: {isrc}' ) if not _is_valid_isrc(isrc): return RuleResult( err_codes.INVALID_ISRC_PATTERN, REJECT, f'Invalid ISRC pattern: {isrc}' ) # Clear known list before next product known_isrcs.clear() def validate_explicit_flag(data: dict, logger: object) -> RuleResult: """Validate Explicit value for release.""" for k, release in data.items(): for k, track in release.tracks.items(): explicit = track.explicit if explicit not in EXPLICIT_CONTENT_TYPES: return RuleResult( err_codes.INVALID_EXPLICIT_FLAG, REJECT, f'Invalid Explicit flag on track: {explicit}' ) def validate_project_code(data: dict, logger: object) -> RuleResult: """Validate project code on a release.""" # Alphanumeric pattern, with dashes and underscores. pattern = '^[A-Za-z0-9_-]*$' for k, release in data.items(): project_code = release.project_code if len(project_code) > 24: return RuleResult( err_codes.INVALID_PROJECT_CODE_LENGTH, REJECT, f'Invalid Project Code; too long: {project_code}' ) if not bool(re.match(pattern, project_code)): return RuleResult( err_codes.INVALID_PROJECT_CODE, REJECT, f'Invalid Project Code; not alphanumeric: {project_code}' ) def validate_file_name(data: dict, logger: object) -> RuleResult: """Validate track file name.""" for k, release in data.items(): for k, track in release.tracks.items(): file_name = track.file_name ext = os.path.splitext(file_name)[1] if ext not in VALID_FILE_NAME_EXTENSIONS: return RuleResult( err_codes.INVALID_FILE_NAME_EXT, REJECT, f'Invalid Track File Name extension: {ext}' ) if len(file_name) > 80: return RuleResult( err_codes.INVALID_FILE_NAME_LENGTH, REJECT, f'Invalid Track File Name length: {file_name}' ) def validate_performer_type(data: dict, logger: object) -> RuleResult: """Validate performer type data.""" for k, release in data.items(): for k, track in release.tracks.items(): p_type = track.performer_1_type if p_type not in PERFORMER_TYPES: return RuleResult( err_codes.INVALID_PERFORMER_TYPE, REJECT, f'Invalid Performer Type: {p_type}' ) def validate_performer_role(data: dict, logger: object) -> RuleResult: """Validate performer main role data.""" for k, release in data.items(): for k, track in release.tracks.items(): p_role = track.performer_1_main_role if p_role not in PERFORMER_MAIN_ROLES: return RuleResult( err_codes.INVALID_PERFORMER_ROLE, REJECT, f'Invalid Performer Role: {p_role}' ) def validate_country_of_recording(data: dict, logger: object) -> RuleResult: """Validate country of recording.""" for k, release in data.items(): for k, track in release.tracks.items(): recording_country = track.country_of_recording if recording_country not in COUNTRY_NAMES: return RuleResult( err_codes.INVALID_COUNTRY_OF_RECORDING, REJECT, f'Invalid country of recording: {recording_country}' ) def validate_nationality_of_original_copyright_owner(data: dict, logger: object) -> RuleResult: # noqa: E501 """Validate nationality of copyright owner.""" for k, release in data.items(): for k, track in release.tracks.items(): country = track.nationality_of_original_copyright_owner if country not in COUNTRY_NAMES: return RuleResult( err_codes.INVALID_COPYRIGHT_OWNER_COUNTRY, REJECT, f'Invalid copyright owner country: {country}' ) def validate_ownership_type(data: dict, logger: object) -> RuleResult: """Validate track ownership type data.""" for k, release in data.items(): for k, track in release.tracks.items(): ownership_type = track.ownership_for_this_sound_recording if ownership_type not in OWNERSHIP_TYPES: return RuleResult( err_codes.INVALID_OWNERSHIP_TYPE, REJECT, f'Invalid track ownership rights: {ownership_type}' ) def validate_publisher(data: dict, logger: object) -> RuleResult: """Validate track ownership type data.""" for k, release in data.items(): for k, track in release.tracks.items(): third_party = track.third_party_publisher publishers = track.publishers if third_party and third_party.lower() == 'yes': if not publishers: return RuleResult( err_codes.INVALID_PUBLISHERS, REJECT, f'Invalid publishers data: {publishers}' ) def validate_track_language(data: dict, logger: object) -> RuleResult: """Validate track audio language data.""" for k, release in data.items(): for k, track in release.tracks.items(): audio_language = track.track_audio_language if audio_language not in LANGUAGE_CODE_MAP: return RuleResult( err_codes.INVALID_TRACK_LANGUAGE, REJECT, f'Invalid track language data: {audio_language}' ) COMMON_RULES = [ validate_mandatory_fields, validate_isrc, require_valid_track_pline, require_valid_cline, require_valid_pline, validate_release_type, validate_artist_country, validate_album_pricing, validate_track_pricing, validate_release_date, validate_sale_start_date, validate_release_meta_language, validate_manufacturer_upc, validate_genre_subgenre, validate_gaps_track_volume_number, validate_explicit_flag, validate_project_code, validate_file_name, validate_performer_type, validate_performer_role, validate_country_of_recording, validate_nationality_of_original_copyright_owner, validate_ownership_type, validate_publisher, validate_track_language, ] # Configure the rules we want to execute per ingestion source RULES_PER_INGESTION_SOURCE = { JSON_FULL: [ *COMMON_RULES, ] }