"""DDEX marshmallow context schema.""" from typing import Dict from ddex_ingester_common.constants.language_codes import ( PRODUCT_LANGUAGE_CODE, TRACK_LANGUAGE_CODE, VIDEO_LANGUAGE_CODE, ) from ddex_ingester_common.helpers.metadata import ( format_lyrics, format_pline, format_publishers, map_explicit_string, map_video_parental_warning, map_video_type, process_language_code, remove_non_alphanumeric_characters, ) from ddex_ingester_common.helpers.xml_document_helper import ( get_conditional_text_value, get_root_element ) from ddex_ingester_common.models.ddex.asset import Asset from ddex_ingester_common.models.ddex.body import Body from ddex_ingester_common.models.ddex.deal import Deal from ddex_ingester_common.models.ddex.deal_term import DealTerm from ddex_ingester_common.models.ddex.genre import Genre from ddex_ingester_common.models.ddex.instant_gratification import ( InstantGratification) from ddex_ingester_common.models.ddex.localized_participant import ( LocalizedParticipant ) from ddex_ingester_common.models.ddex.localized_title import LocalizedTitle from ddex_ingester_common.models.ddex.marketing_highlight import ( MarketingHighlight ) from ddex_ingester_common.models.ddex.p_line import PLine from ddex_ingester_common.models.ddex.participant import Participant from ddex_ingester_common.models.ddex.product import Product from ddex_ingester_common.models.ddex.project import Project from ddex_ingester_common.models.ddex.store import Store from ddex_ingester_common.models.ddex.track import Track from ddex_ingester_common.models.ddex.video import Video from ddex_ingester_common.models.ddex.video_asset import VideoAsset from marshmallow import (EXCLUDE, fields, post_load, pre_load, Schema, validates_schema, ValidationError) from marshmallow.utils import get_value, set_value # https://github.com/marshmallow-code/marshmallow/issues/1311#issuecomment-513478342 class Reach(fields.Field): """Allows for reaching into nested fields.""" def __init__(self, inner, path, **kwargs): """Marshmallow standard constructor.""" super().__init__(**kwargs) self.inner = inner self.path = path def _deserialize(self, value, attr, data, **kwargs): val = get_value(value, self.path) return self.inner.deserialize(val, **kwargs) def _serialize(self, value, attr, obj, **kwargs): val = self.inner._serialize(value, attr, obj, **kwargs) ret = {} set_value(ret, self.path, val) return ret class ReleaseIdSchema(Schema): """Release Id Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE upc = fields.Raw(data_key='ICPN') isrc = fields.String(data_key='ISRC') grid = fields.String(data_key='GRid', missing=None) catalog_number = fields.Dict(data_key='CatalogNumber', missing=None) proprietary_id = fields.Dict(data_key='ProprietaryId', missing=None) @validates_schema def validate_identifiers(self, data, **kwargs): """Enforces validation for the ReleaseId field.""" if not data.get('upc') and not data.get('isrc'): raise ValidationError( 'Neither UPC or ISRC is set in releaseId field') class VideoIdSchema(Schema): """Video Id Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE isrc = fields.String(data_key='ISRC') @validates_schema def validate_identifiers(self, data, **kwargs): """Enforces validation for the VideoId field.""" if not data.get('isrc'): raise ValidationError( 'ISRC is not set in VideoId field') class AssetSchema(Schema): """Asset Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE # TODO: Add Hashsum filename = fields.String(data_key='FileName', missing=None) filepath = fields.String(data_key='FilePath', missing=None) bucket = fields.String(missing=None) key = fields.String(missing=None) ows_assets_filename = fields.String(missing=None) @post_load def make_asset(self, data, **kwargs): """Make Asset model object.""" if not data.get('filename') or not data.get('filepath'): return None return Asset(**data) class LocalizedParticipantSchema(Schema): """Localized Participant Schema.""" language_code = fields.String(required=False, allow_none=True, missing='') name = fields.String(required=False, allow_none=True, missing='') @post_load def make_localized_participant(self, data, **kwargs): """Make Localized Participant model object.""" return LocalizedParticipant(**data) class ParticipantSchema(Schema): """Participant Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE name = fields.String(required=False, allow_none=True, missing='') sequence_number = fields.Int(required=False, missing=None) spotify_uri = fields.String(required=False, missing=None) apple_id = fields.String(required=False, missing=None) roles = fields.List(fields.String(), allow_none=True, missing=None) localized_names = fields.Nested( LocalizedParticipantSchema, many=True, missing=None) @post_load def make_participant(self, data, **kwargs): """Make Participant model object.""" return Participant(**data) class ValidityPeriodSchema(Schema): """ValidityPeriod Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE start_date = fields.String( data_key='StartDate', missing=None ) start_date_time = fields.String( data_key='StartDateTime', missing=None ) @validates_schema def validate_dates(self, data, **kwargs): """Enforces validation for the StartDate/StartDateTime field.""" if not data.get('start_date') and not data.get('start_date_time'): raise ValidationError( 'Neither StartDate or StartDateTime is present in ' 'ValidityPeriod') class InstantGratificationSchema(Schema): """Instant Gratification Schema.""" references = fields.List( fields.String(), required=False, allow_none=True, missing=[], many=True, data_key='DealResourceReference' ) period = fields.Nested( ValidityPeriodSchema, required=False, data_key='Period' ) @post_load def make_instant_grats(self, data, **kwargs): """Make Instant Grats model object.""" formatted_data = { **data.get('period'), **data } return InstantGratification(**formatted_data) class DealTermSchema(Schema): """DealTerm Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE commercial_model_types = fields.List( fields.String(required=False, allow_none=True, missing=''), required=False, allow_none=True, data_key='CommercialModelType', missing=[] ) usage_type = Reach( fields.List(fields.String()), required=False, allow_none=True, missing=[], path='UseType', data_key='Usage', ) territories = fields.List( fields.Raw(required=False), required=False, allow_none=True, data_key='TerritoryCode', missing=[] ) excluded_territories = fields.List( fields.Raw(required=False), required=False, allow_none=True, data_key='ExcludedTerritoryCode', missing=[] ) validity_period = fields.Nested( ValidityPeriodSchema, required=False, data_key='ValidityPeriod' ) takedown = fields.String( required=False, data_key='TakeDown', missing=None, allow_none=True ) pre_order = fields.Boolean( required=False, data_key='IsPreOrderDeal', allow_none=True ) release_display_date = fields.String( required=False, data_key='ReleaseDisplayStartDateTime', allow_none=True ) track_listing_preview_date = fields.String( required=False, data_key='TrackListingPreviewStartDateTime', allow_none=True ) cover_art_preview_date = fields.String( required=False, data_key='CoverArtPreviewStartDateTime', allow_none=True ) clip_preview_date = fields.String( required=False, data_key='ClipPreviewStartDateTime', allow_none=True ) instant_gratifications = fields.Nested( InstantGratificationSchema, required=False, missing=None, data_key='InstantGratificationResourceList' ) price_type = Reach( fields.Dict(), path='PriceType', data_key='PriceInformation', required=False, allow_none=True ) @post_load def make_deal_terms(self, data, **kwargs): """Parse territories that could be string or dictionary.""" territories = [ get_conditional_text_value(code) for code in data.get('territories', []) ] excluded_territories = [ get_conditional_text_value( code) for code in data.get('excluded_territories', []) ] price_type = get_conditional_text_value(data.get('price_type')) """Make Deal Term model object.""" formatted_data = { **data, **data.get('validity_period'), 'territories': territories, 'price_type': price_type, 'excluded_territories': excluded_territories, } return DealTerm(**formatted_data) class DealSchema(Schema): """Deal Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE release_references = fields.List( fields.String(), required=False, allow_none=True, data_key='DealReleaseReference', ) deal_terms = fields.List( Reach(fields.Nested(DealTermSchema, required=False), path='DealTerms'), required=False, allow_none=True, missing=[], data_key='Deal', ) @pre_load def filter_deals_terms_with_invalid_validity_periods( self, data, **kwargs): """Filter out invalid deal terms.""" def has_valid_start_date(deal): if not deal: return False deal_terms = deal.get('DealTerms') if not deal_terms.get('ValidityPeriod'): return False has_valid_start_date = deal_terms and ( 'StartDate' in deal_terms.get('ValidityPeriod', '') or 'StartDateTime' in deal_terms.get('ValidityPeriod', '') ) return has_valid_start_date deals = data.get('Deal') filtered_deals = [deal for deal in deals if has_valid_start_date(deal)] data['Deal'] = filtered_deals return data @post_load def make_deal(self, data, **kwargs): """Make Deal model object.""" return Deal(**data) class StoreSchema(Schema): """Store Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE store_id = fields.String(missing=None, data_key='Id') link = fields.String(missing=None, data_key='Link') delivery_date = fields.String(missing=None, data_key='DeliveryDate') @post_load def make_store(self, data, **kwargs): """Make Store model object.""" return Store(**data) class LocalizedTitleSchema(Schema): """Localized Title Schema.""" language_code = fields.String(required=False, allow_none=True, missing='') title = fields.String(required=False, allow_none=True, missing='') version = fields.String(missing=None) @post_load def make_localized_title(self, data, **kwargs): """Make Localized Title model object.""" return LocalizedTitle(**data) class TrackSchema(Schema): """Track Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE release_id = fields.Nested( ReleaseIdSchema, data_key='ReleaseId', required=False ) release_reference = fields.String( data_key='ReleaseReference', required=False, allow_none=True, missing='' ) release_type = fields.String() sequence_number = fields.Int(required=False, missing=None) volume = fields.Int(required=False, missing=None) asset = fields.Nested(AssetSchema, missing=None) track_name = fields.String(required=False, allow_none=True, missing='') track_version = fields.String(allow_none=True, missing=None) lyrics = fields.String(missing=None) lyrics_language = fields.String(allow_none=True, missing=None) explicit = fields.String(allow_none=True, missing=None) p_line = fields.String(allow_none=True, missing=None) c_line = fields.String(allow_none=True, missing=None) ownership_rights = fields.String(allow_none=True, missing=None) recording_country_code = fields.String(allow_none=True, missing=None) us_publishing_obligation = fields.String(allow_none=True, missing=None) publishers = fields.List(fields.String(), allow_none=True, missing=None) display_artists = fields.Nested( ParticipantSchema, many=True, missing=None) resource_contributors = fields.Nested( ParticipantSchema, many=True, missing=None) isrc = fields.String(missing=None) copyright_owner_country = fields.String(missing=None) localized_titles = fields.Nested( LocalizedTitleSchema, many=True, missing=None) preview_start_time = fields.Int(missing=None) resource_reference = fields.String( required=False, allow_none=True, missing='' ) content_type = fields.String(allow_none=True, missing=None) @post_load def make_track(self, data, **kwargs): """Make Track model object.""" formatted_data = { 'tuid': None, 'release_reference': data.get('release_reference'), 'isrc': data.get('release_id', {}).get('isrc'), 'release_type': data.get('release_type'), 'sequence_number': data.get('sequence_number'), 'volume': data.get('volume'), 'asset': data.get('asset'), 'track_name': data.get('track_name'), 'track_version': data.get('track_version'), 'lyrics': data.get('lyrics'), 'lyrics_language': data.get('lyrics_language'), 'explicit': data.get('explicit'), 'p_line': data.get('p_line'), 'c_line': data.get('c_line'), 'ownership_rights': data.get('ownership_rights'), 'recording_country_code': data.get('recording_country_code'), 'us_publishing_obligation': data.get('us_publishing_obligation'), 'publishers': data.get('publishers'), 'display_artists': data.get('display_artists'), 'resource_contributors': data.get('resource_contributors'), 'copyright_owner_country': data.get('copyright_owner_country'), 'localized_titles': data.get('localized_titles'), 'preview_start_time': data.get('preview_start_time'), 'resource_reference': data.get('resource_reference'), 'content_type': data.get('content_type'), } return Track(**formatted_data) class GenreSchema(Schema): """Genre schema.""" genre = fields.String(required=False, allow_none=True, missing='') subgenre = fields.String(required=False, missing=None) @post_load def make_genre(self, data, **kwargs): """Make genre model object.""" return Genre(**data) class PLineSchema(Schema): """PLine schema.""" line = fields.String(required=False, allow_none=True, missing='') year = fields.Int(required=False, missing=None) @post_load def make_genre(self, data, **kwargs): """Make pline model object.""" return PLine(**data) class VideoAssetSchema(Schema): """VideoAsset Schema.""" filename = fields.String(missing=None) filepath = fields.String(missing=None) bucket = fields.String(missing=None) key = fields.String(missing=None) aspect_ratio = fields.String(missing=None) @post_load def make_video_asset(self, data, **kwargs): """Make Asset model object.""" if not data.get('filename') or not data.get('filepath'): return None return VideoAsset(**data) class VideoSchema(Schema): """Video Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE release_id = fields.Nested( ReleaseIdSchema, data_key='ReleaseId', required=False ) release_reference = fields.String( data_key='ReleaseReference', required=False, allow_none=True, missing='' ) release_type = fields.String() assets = fields.Nested(VideoAssetSchema, missing=None, many=True) video_name = fields.String(required=False, allow_none=True, missing='') video_version = fields.String(allow_none=True, missing=None) explict = fields.String(allow_none=True, missing=None) p_line = fields.Nested(PLineSchema, allow_none=True, missing=None) c_line = fields.Nested(PLineSchema, allow_none=True, missing=None) display_artists = fields.Nested(ParticipantSchema, many=True, missing=None) resource_contributors = fields.Nested( ParticipantSchema, many=True, allow_none=True, missing=None) title_language = fields.String(allow_none=True, missing=None) content_language = fields.String(allow_none=True, missing=None) product_code = fields.String(allow_none=True, missing=None) imprint = fields.String(allow_none=True, missing=None) video_type = fields.String(allow_none=True, missing=None) genres = fields.Nested( GenreSchema, many=True, allow_none=True, missing=None) associated_track_isrc = fields.String(allow_none=True, missing=None) description = fields.String(allow_none=True, missing=None) parental_advisory = fields.String(allow_none=True, missing=None) keywords = fields.List(fields.String(), allow_none=True, missing=None) lyrics = fields.String(missing=None) content_type = fields.String(missing=None) @post_load def make_video(self, data, **kwargs): """Make Video model object.""" formatted_data = { 'tuid': None, 'release_reference': data.get('release_reference'), 'isrc': data.get('release_id', {}).get('isrc'), 'release_type': data.get('release_type'), 'assets': data.get('assets'), 'video_name': data.get('video_name'), 'video_version': data.get('video_version'), 'explicit': data.get('explict'), 'p_line': data.get('p_line'), 'c_line': data.get('c_line'), 'display_artists': data.get('display_artists'), 'resource_contributors': data.get('resource_contributors'), 'title_language': data.get('title_language'), 'content_language': data.get('content_language'), 'product_code': data.get('product_code'), 'imprint': data.get('imprint'), 'video_type': data.get('video_type'), 'genres': data.get('genres'), 'associated_track_isrc': data.get('associated_track_isrc'), 'description': data.get('description'), 'parental_advisory': data.get('parental_advisory'), 'keywords': data.get('keywords'), 'lyrics': data.get('lyrics'), 'content_type': data.get('content_type'), } return Video(**formatted_data) class ProductSchema(Schema): """Product Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE release_id = fields.Nested( ReleaseIdSchema, data_key='ReleaseId', required=False) release_reference = fields.String( data_key='ReleaseReference', required=False, allow_none=True, missing='') original_release_date = fields.String(required=False, missing=None) is_main_release = fields.Boolean(data_key='@IsMainRelease', missing=False) release_type = fields.String() artwork = fields.Nested(AssetSchema, missing=None) vendor_id = fields.Int(missing=None) subaccount_id = fields.Int(missing=None) display_artist_name = fields.String() product_name = fields.String(required=False, allow_none=True, missing='') status = fields.String(missing=None) display_artists = fields.Nested( ParticipantSchema, many=True, missing=None) metadata_language = fields.String(missing=None) product_version = fields.String(missing=None) genres = fields.Nested(GenreSchema, required=False, missing=[], many=True) p_line = fields.String(missing=None) c_line = fields.String(missing=None) marketing_blurb = fields.String(missing=None) stores = fields.Nested(StoreSchema, missing=None, many=True) artist_profile_page = fields.String(missing=None) localized_titles = fields.Nested( LocalizedTitleSchema, many=True, missing=None) imprint = fields.String(allow_none=True, missing=None) not_for_distribution = fields.String(allow_none=True, missing=None) special_instructions = fields.String(allow_none=True, missing=None) product_version_notes = fields.String(allow_none=True, missing=None) proprietary_id = fields.String(allow_none=True, missing=None) @post_load def make_product(self, data, **kwargs): """Make Product model object.""" if data.get('is_main_release'): formatted_data = { 'product_id': None, 'release_reference': data.get('release_reference'), 'upc': get_conditional_text_value(data.get('release_id', {}) .get('upc')), 'is_main_release': data.get('is_main_release'), 'release_type': data.get('release_type'), 'original_release_date': data.get('original_release_date'), 'artwork': data.get('artwork'), 'grid': data.get('release_id').get('grid'), 'catalog_number': get_conditional_text_value( data.get('release_id', {}).get('catalog_number')), 'display_artist_name': data.get('display_artist_name'), 'product_name': data.get('product_name'), 'status': data.get('status'), 'display_artists': data.get('display_artists'), 'metadata_language': data.get('metadata_language'), 'product_version': data.get('product_version'), 'genres': data.get('genres'), 'p_line': data.get('p_line'), 'c_line': data.get('c_line'), 'marketing_blurb': data.get('marketing_blurb'), 'stores': data.get('stores'), 'artist_profile_page': data.get('artist_profile_page'), 'localized_titles': data.get('localized_titles'), 'vendor_id': data.get('vendor_id'), 'subaccount_id': data.get('subaccount_id'), 'imprint': data.get('imprint'), 'not_for_distribution': data.get('not_for_distribution'), 'special_instructions': data.get('special_instructions'), 'product_version_notes': data.get('product_version_notes'), 'proprietary_id': get_conditional_text_value( data.get('release_id', {}).get('proprietary_id')) } return Product(**formatted_data) class MarketingHighlightSchema(Schema): """Marketing Highlight Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE highlight = fields.String( required=False, missing=None, data_key='Highlight' ) store_codes = fields.List( Reach(fields.String(), allow_none=True, path='Id'), required=False, allow_none=True, missing=None, data_key='Store' ) territory_codes = fields.List( fields.String(), required=False, allow_none=True, missing=None, data_key='TerritoryCode' ) @post_load def make_highlights(self, data, **kwargs): """Make Project model object.""" return MarketingHighlight(**data) class ProjectSchema(Schema): """Project Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE name = fields.String(required=False, missing=None) project_code = fields.String(required=False, missing=None) description = fields.String(required=False, missing=None) artist = fields.Nested(ParticipantSchema, missing=None) marketing_highlights = fields.Nested( MarketingHighlightSchema, many=True, missing=None ) @post_load def make_project(self, data, **kwargs): """Make Project model object.""" formatted_data = { 'description': data.get('description'), 'name': data.get('name'), 'project_code': data.get('project_code'), 'marketing_highlights': data.get('marketing_highlights'), 'artist': data.get('artist') } return Project(**formatted_data) class DDEXSchema(Schema): """DDEX Schema.""" class Meta: """Marshmallow Meta class.""" unknown = EXCLUDE project = fields.Nested(ProjectSchema, missing=None) product = fields.Nested(ProductSchema, required=False) tracks = fields.Nested( TrackSchema, required=False, many=True, allow_none=True) video = fields.Nested(VideoSchema, required=False, allow_none=True) deals = fields.Nested(DealSchema, required=False, missing=[], many=True) bucket = fields.String(missing=None) key = fields.String(missing=None) message_id = fields.String(required=False, allow_none=True, missing='') message_thread_id = fields.String(required=False, missing=None) party_id = fields.String(required=False, allow_none=True, missing='') update_indicator = fields.String( required=False, missing=None, ) execution_name = fields.String(allow_none=True) state_machine_name = fields.String(allow_none=True) execution_start_time = fields.String(allow_none=True) @pre_load def preprocess(self, data, **kwargs): """Alter the data passed to ``load()`` before Marshmallow attempts deserialization.""" # noqa # Get document data irrespective if sme or awal data = get_root_element(data) # Get Product and enrich with asset information release_list = get_value( data, 'ReleaseList.Release', [] ) artwork = get_value( data, ('ResourceList.Image.' 'ImageDetailsByTerritory.TechnicalImageDetails.File') ) sound_recordings = get_value( data, 'ResourceList.SoundRecording', [] ) video = get_value( data, 'ResourceList.Video', None ) project = get_value( data, 'Project', None ) extended_product = get_value( data, 'Product', None ) extended_tracks = get_value( data, 'Tracks', None ) main_product = self.get_main_product(release_list) release_detail = self.get_details_by_territory( main_product, 'ReleaseDetailsByTerritory' ) original_release_date = get_value( main_product, 'GlobalOriginalReleaseDate' ) if not original_release_date: original_release_date = get_value( release_detail, 'OriginalReleaseDate' ) product_version = get_value( main_product, 'ReferenceTitle.SubTitle' ) release_type = get_conditional_text_value( get_value(main_product, 'ReleaseType') ) metadata_language_code = get_value( main_product, 'ReferenceTitle.@LanguageAndScriptCode' ) metadata_language = process_language_code( metadata_language_code, PRODUCT_LANGUAGE_CODE ) display_artists = self.get_display_artists( release_detail, metadata_language_code) genres = self.get_genres(release_detail) p_line = format_pline( get_value( main_product, 'PLine.PLineText' ), get_value( main_product, 'PLine.Year', None ) ) c_line = format_pline( get_value( main_product, 'CLine.CLineText' ), get_value( main_product, 'CLine.Year', None ) ) marketing_blurb = get_value( extended_product, 'MarketingBlurb' ) store = get_value( extended_product, 'Stores.Store' ) track_cline, track_pline = None, None if sound_recordings: recording_territory = self.get_details_by_territory( sound_recordings[0], 'SoundRecordingDetailsByTerritory' ) track_cline = format_pline( get_value( recording_territory, 'CLine.CLineText', None ), get_value( recording_territory, 'CLine.Year', None ) ) track_pline = format_pline( get_value( recording_territory, 'PLine.PLineText', None ), get_value( recording_territory, 'PLine.Year', None ) ) chosen_product_cline = self.get_product_c_line( c_line, p_line, track_cline, track_pline) localized_titles = self.get_localized_titles( metadata_language_code, get_value(release_detail, 'Title') ) data['product'] = { **main_product, 'release_type': release_type, 'artwork': artwork, 'display_artist_name': get_value( release_detail, 'DisplayArtistName' ), 'product_name': get_value( main_product, 'ReferenceTitle.TitleText' ), 'original_release_date': original_release_date, 'display_artists': display_artists, 'product_version': product_version, 'metadata_language': metadata_language, 'genres': genres, 'p_line': p_line, 'c_line': chosen_product_cline, 'marketing_blurb': marketing_blurb, 'stores': store, 'artist_profile_page': get_value( main_product, 'ArtistProfilePage.PageName' ), 'localized_titles': localized_titles, 'vendor_id': get_value( extended_product, 'VendorId', None ), 'subaccount_id': get_value( extended_product, 'SubaccountId', None ), 'imprint': self.get_imprint( get_value(release_detail, 'LabelName') ), 'not_for_distribution': get_value( extended_product, 'NotForDistribution', None ), 'special_instructions': get_value( extended_product, 'SpecialInstructions', None ), 'product_version_notes': get_value( extended_product, 'ProductVersionNotes', None ), } if not data['product']['ReleaseId'].get('CatalogNumber'): data['product']['ReleaseId']['CatalogNumber'] = get_value( extended_product, 'ProductId.CatalogNumber', None ) description = get_value( project, 'Description' ) name = get_value( project, 'Name' ) project_code = get_value( project, 'Id' ) marketing_highlights = get_value( project, 'MarketingHighlights.MarketingHighlight' ) if project: project_artist = self.construct_project_artist(project) data['project'] = { 'description': description, 'name': name, 'project_code': project_code, 'marketing_highlights': marketing_highlights, 'artist': project_artist } # Get Tracks and enrich with asset information data['tracks'] = self.get_tracks( data, release_list, sound_recordings, extended_tracks ) data['video'] = self.get_video( data, release_list, video, extended_tracks ) data['deals'] = get_value( data, 'DealList.ReleaseDeal' ) data['message_id'] = get_value( data, 'MessageHeader.MessageId' ) data['message_thread_id'] = get_value( data, 'MessageHeader.MessageThreadId' ) data['update_indicator'] = get_value( data, 'UpdateIndicator', ) # PartyId is always a list because there are multiple artist PartyIds # There is only one MessageSender PartyId but the fields have the same name # noqa party_ids = get_value(data, 'MessageHeader.MessageSender.PartyId') if party_ids: data['party_id'] = get_conditional_text_value(party_ids[0]) data = self.handle_incomplete_localizations(data) return data @post_load def make_body(self, data, **kwargs): """Make Context model object.""" return Body(**data) def get_main_product(self, release_list) -> dict: """Get main release product out of DDEX.""" product = next( (release for release in release_list if self.is_main_release(release)), None) return product def get_tracks( self, data, releases, sound_recordings, extended_tracks_data ) -> list: """Pull tracks together with their relevant fields.""" sound_recording_metadata = {} for sound_recording in sound_recordings: # Map metadata of each sound recording to its ISRC isrc = get_value(sound_recording, 'SoundRecordingId.ISRC') extended_track = self.get_extended_track_data( isrc, extended_tracks_data ) metadata_language = get_value( sound_recording, 'ReferenceTitle.@LanguageAndScriptCode' ) recording_territory = self.get_details_by_territory( sound_recording, 'SoundRecordingDetailsByTerritory') sound_recording_metadata[isrc] = { 'track_name': get_value( sound_recording, 'ReferenceTitle.TitleText'), 'track_version': get_value( sound_recording, 'ReferenceTitle.SubTitle'), 'lyrics_language': process_language_code( get_value(sound_recording, 'LanguageOfPerformance', None), TRACK_LANGUAGE_CODE ), 'explicit': map_explicit_string(get_value( recording_territory, 'ParentalWarningType' )), 'p_line': format_pline( get_value( recording_territory, 'PLine.PLineText' ), get_value( recording_territory, 'PLine.Year' ) ), 'c_line': format_pline( get_value( recording_territory, 'CLine.CLineText' ), get_value( recording_territory, 'CLine.Year' ) ), 'display_artists': self.get_display_artists( recording_territory, metadata_language, ), 'resource_contributors': self.get_resource_contributors( recording_territory ), 'lyrics': format_lyrics(extended_track.get('Lyrics')), 'ownership_rights': extended_track.get('OwnershipRights'), 'recording_country_code': extended_track.get( 'RecordingCountry' ), 'copyright_owner_country': extended_track.get( 'CopyrightOwnerCountry' ), 'publishers': format_publishers(get_value(extended_track, 'Publisher', [])), 'us_publishing_obligation': extended_track.get( 'USPublishingObligation' ), 'localized_titles': self.get_localized_titles( metadata_language, get_value( recording_territory, 'Title' ) ), 'resource_reference': get_value( sound_recording, 'ResourceReference' ), 'content_type': extended_track.get('ContentType'), } tracks = [] for release in releases: release_type_node = release.get('ReleaseType') release_type = get_conditional_text_value(release_type_node) if not release_type: rid = get_value(release, 'ReleaseId') raise ValidationError( f'Missing Release Type for release with Release Id: {rid}') if release_type == 'TrackRelease': isrc = get_value(release, 'ReleaseId.ISRC') tracks.append({ **release, **sound_recording_metadata[isrc], 'release_type': release_type, 'asset': self.get_track_asset(data, release), 'sequence_number': self.get_track_sequence(data, release), 'volume': self.get_track_side(data, release), 'preview_start_time': self.get_track_preview_start_time(data, release), }) return tracks def get_extended_track_data(self, isrc, extended_tracks_data): """Get extended track data by ISRC.""" found = {} if extended_tracks_data: tracks = extended_tracks_data.get('Track', {}) for track in tracks: if track.get('ISRC') == isrc: found = track return found def get_track_asset(self, data, track) -> dict: """Get track asset information.""" sound_recording = self.get_track_sound_recording( data, track ) sound_recording_details = get_value( self.get_details_by_territory( sound_recording, 'SoundRecordingDetailsByTerritory'), 'TechnicalSoundRecordingDetails', [] ) return next( (detail.get('File') for detail in sound_recording_details if detail.get('IsPreview') == 'false'), None ) def get_track_preview_start_time(self, data, track) -> dict: """Get track preview start time.""" sound_recording = self.get_track_sound_recording( data, track ) sound_recording_details = get_value( self.get_details_by_territory( sound_recording, 'SoundRecordingDetailsByTerritory'), 'TechnicalSoundRecordingDetails', [] ) return next(( detail.get('PreviewDetails').get('StartPoint') for detail in sound_recording_details if detail.get('IsPreview') == 'true'), None ) def get_track_sound_recording(self, data, track) -> dict: """Get track associated sound recording.""" sound_recordings = get_value( data, 'ResourceList.SoundRecording', [] ) asset_reference = get_conditional_text_value(get_value( track, 'ReleaseResourceReferenceList.ReleaseResourceReference' )) sound_recording = [ d for d in sound_recordings if d.get('ResourceReference') == asset_reference ] if sound_recording: return sound_recording[0] return None def get_resource_groups(self, data) -> list: """Get resource groups.""" product = self.get_main_product(get_value( data, 'ReleaseList.Release', [] )) release_detail = self.get_details_by_territory( product, 'ReleaseDetailsByTerritory' ) resource_groups = get_value( release_detail, 'ResourceGroup.ResourceGroup', [] ) # This field can be a list or a dict. Make it a list so we process # everything the same way if isinstance(resource_groups, dict): resource_groups = [resource_groups] return resource_groups def get_details_by_territory( self, root, list_name, target_territory='Worldwide') -> dict: """Filter a possible list of details by desired territory.""" list_details = get_value( root, list_name, [] ) # Failed to find details, return None for marshmallow to handle if not list_details: return None # We've got something. Is it a list? if type(list_details) != list: return list_details # We only have one detail so return it if len(list_details) == 1: return list_details[0] # Find a territory that equals our target territory for detail in list_details: territories = get_value( detail, 'TerritoryCode', [] ) for territory in territories: if get_conditional_text_value(territory) == target_territory: return detail # Return the first territory if we fail to find one return list_details[0] def get_participants( self, release_detail, main_field, role_field, product_language='') -> list: """Get display artists.""" display_artists = [] artist_list = get_value( release_detail, main_field, [] ) sequence_number_count = 1 for artist in artist_list: party_id_list = get_value(artist, 'PartyId') spotify_uri, apple_id = self.get_artist_ids(party_id_list) party_name_list = get_value(artist, 'PartyName') localized_names = [] for party_name in party_name_list: language_code = get_value(party_name, '@LanguageAndScriptCode') if language_code and language_code != product_language: localized_names.append({ 'name': get_value(party_name, 'FullName', None), 'language_code': language_code.upper(), }) else: name = get_value(party_name, 'FullName', None) sequence_number = artist.get('@SequenceNumber') if not sequence_number: sequence_number = sequence_number_count sequence_number_count += 1 roles = get_value(artist, role_field, []) roles_list = [] for role in roles: role_text = get_conditional_text_value(role) roles_list.append( remove_non_alphanumeric_characters(role_text)) display_artists.append( { 'name': name, 'sequence_number': sequence_number, 'roles': roles_list, 'localized_names': localized_names, 'spotify_uri': spotify_uri, 'apple_id': apple_id, } ) return display_artists def get_track_sequence(self, data, track) -> int: """Get track sequence number.""" resource_groups = self.get_resource_groups(data) reference = self.get_track_resource_reference(track) sequence_number = None for group in resource_groups: content_items = self.get_content_items(group) for content_item in content_items: content_reference = get_conditional_text_value( get_value(content_item, 'ReleaseResourceReference')) if content_reference == reference: sequence_number = content_item.get('SequenceNumber') return sequence_number def get_track_side(self, data, track) -> int: """Get track side.""" resource_groups = self.get_resource_groups(data) reference = self.get_track_resource_reference(track) side = None for group in resource_groups: content_items = self.get_content_items(group) for content_item in content_items: content_reference = get_conditional_text_value( get_value(content_item, 'ReleaseResourceReference')) if content_reference == reference: side = group.get('SequenceNumber') return side def get_content_items(self, group) -> list: """Get content items out of a resource group.""" if not group: return [] if isinstance(group, list): # There's multiple ResourceGroup items at this level content_items = [] for item in group: content_items.extend([ *item.get('ResourceGroupContentItem', []), *self.get_content_items(item.get('ResourceGroup')) ]) return content_items else: return [ *group.get('ResourceGroupContentItem', []), *self.get_content_items(group.get('ResourceGroup')) ] def get_track_resource_reference(self, track) -> str: """Get track resource reference.""" return get_conditional_text_value(get_value( track, 'ReleaseResourceReferenceList.ReleaseResourceReference')) def get_video(self, data, releases, video_tags, extended_tracks) -> dict: """Pull video together with their relevant fields.""" if not video_tags: return None if type(video_tags) == list: raise ValidationError('Unexpected Video tag found.') video_isrc = get_value(video_tags, 'VideoId.ISRC') video_metadata = self.map_video_metadata(video_tags) video_track_release = {} if extended_tracks: extended_track = self.get_extended_track_data( video_isrc, extended_tracks) video_metadata['lyrics'] = format_lyrics( extended_track.get('Lyrics')) video_metadata['content_type'] = extended_track.get('ContentType') for release in releases: release_details_by_territory = get_value( release, 'ReleaseDetailsByTerritory' )[0] if self.is_main_release(release): video_metadata['description'] = get_value( release_details_by_territory, 'Synopsis' ) video_metadata['keywords'] = get_value( release_details_by_territory, 'Keywords' ) if not video_metadata.get('genres'): video_metadata['genres'] = self.get_genres( release_details_by_territory) if not video_metadata.get('title_language'): video_metadata['title_language'] = process_language_code( get_value(release, 'ReferenceTitle.@LanguageAndScriptCode'), VIDEO_LANGUAGE_CODE ) # Overwrite as release pline and cline have priority over the values in video details # noqa # Default in case one of them is missing self.populate_p_c_line( release, video_metadata, default=True ) release_type_node = release.get('ReleaseType') release_type = get_conditional_text_value(release_type_node) if release_type == 'VideoTrackRelease': video_track_release = release video_metadata['product_code'] = get_conditional_text_value( release['ReleaseId'].get('CatalogNumber')) related_releases = get_value( release_details_by_territory, 'RelatedRelease', [] ) for related_release in related_releases: if get_value(related_release, 'ReleaseRelationshipType') == 'IsFromAudio': # noqa video_metadata['associated_track_isrc'] = get_value( related_release, 'ReleaseId.ISRC' ) return { **video_track_release, **video_metadata, 'release_type': release_type, 'assets': self.get_video_assets(data) } def map_video_metadata(self, video) -> dict: """Map metadata of each video to its ISRC.""" video_details_by_territory = self.get_details_by_territory( video, 'VideoDetailsByTerritory') video_metadata = { 'video_name': get_value( video, 'ReferenceTitle.TitleText'), 'video_version': get_value( video, 'ReferenceTitle.SubTitle'), 'title_language': process_language_code( get_value(video, 'ReferenceTitle.@LanguageAndScriptCode'), VIDEO_LANGUAGE_CODE ), 'content_language': process_language_code( get_value(video, 'LanguageOfPerformance'), VIDEO_LANGUAGE_CODE ), 'explicit': map_explicit_string(get_value( video_details_by_territory, 'ParentalWarningType' )), 'display_artists': self.get_display_artists( video_details_by_territory), 'resource_contributors': self.get_resource_contributors( video_details_by_territory ), 'imprint': self.get_imprint( get_value(video_details_by_territory, 'LabelName') ), 'video_type': map_video_type( get_conditional_text_value(get_value(video, 'VideoType'))), 'genres': self.get_genres( video_details_by_territory ), 'parental_advisory': map_video_parental_warning(get_value( video_details_by_territory, 'ParentalWarningType' )), } # Don't default as that will be done later when checking the release self.populate_p_c_line( video_details_by_territory, video_metadata, default=False ) return video_metadata def get_video_assets(self, data) -> list: """Get video asset information.""" video_tag = get_value( data, 'ResourceList.Video', [] ) video_details = get_value( self.get_details_by_territory( video_tag, 'VideoDetailsByTerritory'), 'TechnicalVideoDetails', [] ) return [ { 'filename': detail.get('File', {}).get('FileName'), 'filepath': detail.get('File', {}).get('FilePath'), 'aspect_ratio': get_conditional_text_value( detail.get('AspectRatio')) } for detail in video_details if detail.get('IsPreview') == 'false' ] def construct_project_artist(self, data) -> dict: """Get project artist and store links.""" if not get_value(data, 'Artist', None): # We want to catch missing project artists in a validation rule. return project_artist = { 'name': None, 'spotify_uri': None, 'apple_id': None, 'sequence_number': 1 } project_artist['name'] = get_value( data, ('Artist.Name') ) artist_ids = get_value( data, ('Artist.ArtistId') ) spotify_uri, apple_id = self.get_artist_ids(artist_ids) project_artist['spotify_uri'] = spotify_uri project_artist['apple_id'] = apple_id return project_artist def get_artist_ids(self, artist_ids): """Determine and return the store type IDs for artist ID List.""" spotify_uri = None apple_id = None apple_id_2 = None if artist_ids: for artist_id in artist_ids: namespace = artist_id.get('@Namespace', '').lower() if namespace == 'spotify': spotify_uri = get_conditional_text_value(artist_id) elif namespace == 'itunes': apple_id = get_conditional_text_value(artist_id) # Some deliveries have the namespace Apple instead of Itunes # Itunes is the correct namespace but we also support Apple elif namespace == 'apple': apple_id_2 = get_conditional_text_value(artist_id) apple_id = apple_id if apple_id else apple_id_2 return spotify_uri, apple_id def get_localized_titles(self, metadata_language, localized_titles): """Get list of localized track titles.""" processed_localized_titles = [] if localized_titles: for localized_title in localized_titles: localized_language_code = \ localized_title.get('@LanguageAndScriptCode') if localized_title.get('@TitleType') == 'FormalTitle' and \ localized_language_code and \ localized_language_code != metadata_language: processed_localized_title = { 'language_code': localized_language_code.upper(), 'title': localized_title.get('TitleText'), 'version': localized_title.get('SubTitle') } processed_localized_titles.append( processed_localized_title) return processed_localized_titles def get_product_c_line(self, prod_cline, prod_pline, track_cline, track_pline): """Decides product CLine based on based available data. Priority: #1: Product CLine #2: Product Pline #3: Track 1 CLine #4: Track 1 PLine """ c_line = prod_cline if prod_cline else prod_pline \ if prod_pline else track_cline if track_cline \ else track_pline if track_pline else None return format_pline(c_line) if c_line else None def get_genres(self, release_detail) -> list: """Get genres from main release detail.""" genre_node = get_value( release_detail, 'Genre' ) genres = [] for genre in genre_node or []: if genre: genres.append({ 'genre': genre.get('GenreText'), 'subgenre': genre.get('SubGenre') }) return genres def get_display_artists(self, release_detail, product_language='') -> list: """Get display artists from main release detail.""" return self.get_participants( release_detail, 'DisplayArtist', 'ArtistRole', product_language ) def get_resource_contributors(self, release_detail) -> list: """Get resource contributors from main release detail.""" resource_contributors = self.get_participants( release_detail, 'ResourceContributor', 'ResourceContributorRole', ) indirect_resource_contributors = self.get_participants( release_detail, 'IndirectResourceContributor', 'IndirectResourceContributorRole', ) all_contributors = {} for contributor in resource_contributors: name = contributor['name'] # Check for repeated contributors if name in all_contributors: existing_roles = all_contributors[name]['roles'] all_contributors[name]['roles'] = list(set( existing_roles + contributor['roles'])) else: all_contributors[name] = contributor highest_sequence_number = 0 if all_contributors: highest_sequence_number = max( int(contributor['sequence_number']) for contributor in all_contributors.values() ) for indirect_contributor in indirect_resource_contributors: name = indirect_contributor['name'] # Check for repeated contributors if name in all_contributors: existing_roles = all_contributors[name]['roles'] all_contributors[name]['roles'] = list(set( existing_roles + indirect_contributor['roles'])) else: # Avoid conflict in sequence numbers sequence_number = int(indirect_contributor['sequence_number']) indirect_contributor['sequence_number'] = \ sequence_number + highest_sequence_number all_contributors[name] = indirect_contributor return list(all_contributors.values()) def handle_incomplete_localizations(self, data) -> dict: """Deal with missing or incomplete localization data in the DDEX. This function completes missing localization data by using the original title or artist name as defaults The goal is to have a localized title for the product and tracks as well as a localized name for every display artist """ language_codes = self.get_all_language_codes(data) if not language_codes: return data self.add_default_localized_titles( data['product'], 'product', language_codes ) self.add_default_localized_names( data['product'], language_codes ) if data['tracks']: for track in data['tracks']: self.add_default_localized_titles( track, 'track', language_codes ) self.add_default_localized_names( track, language_codes ) return data def get_all_language_codes( self, data: dict) -> set: """Get language codes of all the localizations in the DDEX.""" language_codes = set() language_codes |= self.get_language_codes(data['product']) if data.get('tracks'): for track in data['tracks']: language_codes |= self.get_language_codes(track) return language_codes def get_language_codes( self, data: dict) -> set: """Get language codes of the localizations of a track or product.""" language_codes = set() if data.get('localized_titles'): for localized_title in data['localized_titles']: language_codes.add(localized_title['language_code']) if data.get('display_artists'): for display_artist in data['display_artists']: for localized_name in display_artist['localized_names']: language_codes.add(localized_name['language_code']) return language_codes def add_default_localized_titles( self, data: dict, track_or_product: str, language_codes: set): """Complete localization data with original track or product title.""" if not data.get('localized_titles'): data['localized_titles'] = [] missing_codes = language_codes.copy() for localized_title in data['localized_titles']: language_code = localized_title['language_code'] if language_code not in missing_codes: raise Exception(f'Duplicate language code {language_code}') missing_codes.remove(language_code) default_title = { 'title': data.get(track_or_product + '_name'), 'version': data.get(track_or_product + '_version') } for code in missing_codes: data['localized_titles'].append({ **default_title, 'language_code': code }) def add_default_localized_names( self, data: dict, language_codes: set): """Use original display artist name to complete localization data.""" if data.get('display_artists'): for display_artist in data['display_artists']: if not display_artist.get('localized_names'): display_artist['localized_names'] = [] missing_codes = language_codes.copy() for localized_name in display_artist['localized_names']: language_code = localized_name['language_code'] if language_code not in missing_codes: raise Exception(f'Duplicate language code {language_code}') # noqa missing_codes.remove(language_code) for code in missing_codes: display_artist['localized_names'].append({ 'name': display_artist['name'], 'language_code': code }) def is_main_release(self, release: Dict) -> bool: """Return if this release is the main release.""" return release and release.get('@IsMainRelease') == 'true' def get_imprint(self, label_names: Dict) -> str: """Get label imprint.""" for label in label_names or []: if isinstance(label, dict) and \ label.get('@LabelNameType') == 'DisplayLabelName': return get_conditional_text_value(label) if label_names: return get_conditional_text_value(label_names[0]) return None def populate_p_c_line(self, release, video_metadata, default=True): """Populate p line and c line on provided dict. If there is p or c line data this function overwrites existing values The default flag populates a missing value with the other if possible """ full_p_line = format_pline( get_value( release, 'PLine.PLineText' ), get_value( release, 'PLine.Year', None ) ) p_line_year, p_line = \ full_p_line.split(' ', 1) if full_p_line else (None, None) full_c_line = format_pline( get_value( release, 'CLine.CLineText' ), get_value( release, 'CLine.Year', None ) ) c_line_year, c_line = \ full_c_line.split(' ', 1) if full_c_line else (None, None) if full_p_line: video_metadata['p_line'] = {} video_metadata['p_line']['line'] = p_line video_metadata['p_line']['year'] = p_line_year if full_c_line: video_metadata['c_line'] = {} video_metadata['c_line']['line'] = c_line video_metadata['c_line']['year'] = c_line_year if default: if not video_metadata['c_line'] and video_metadata['p_line']: video_metadata['c_line'] = video_metadata['p_line'] if not video_metadata['p_line'] and video_metadata['c_line']: video_metadata['p_line'] = video_metadata['c_line']