import pandas as pd import datetime import string import math import re from constants import knr_consts from utils.validation import valid_str_duration, is_empty_value def format_duration(duration): if valid_str_duration(duration): return duration h_m_s = str(datetime.timedelta(seconds=round(float(duration)))) if len(h_m_s.split(":")[0]) == 1: return "0" + h_m_s else: return h_m_s def format_date(col): return pd.to_datetime(col, dayfirst=True, format="mixed").dt.strftime('%d-%b-%Y') # sales start date can be empty for video content def format_video_date(date): return pd.to_datetime(date, dayfirst=True, errors="coerce", format="mixed").dt.strftime('%d-%b-%Y') def strip_all_strings(df): return df.applymap(lambda x: x.strip() if isinstance(x, str) else x) def format_contributor_types(types): if is_empty_value(types): return '' types_array = str(types).strip().split('|') formatted_types = [] for item in types_array: # Only valid items will be appended to formatted_types, invalid string values, extra pipes, and empty strings are ignored x = item.upper().translate(str.maketrans('', '', f'{string.punctuation} ')).strip() if x in knr_consts.MAIN_PERFORMER_CATEGORY_MAPS: formatted_types.append('Main Performer') elif x in knr_consts.FEATURING_PERFORMER_CATEGORY_MAPS: formatted_types.append('Featuring Performer') elif x in knr_consts.SESSION_MUSICIAN_CATEGORY_MAPS: formatted_types.append('Session Musician') piped_types = '|'.join(formatted_types) return piped_types def format_contributor_names(names): if is_empty_value(names): return '' names_array = list(filter(lambda x: x != "", [x.strip() for x in names.split('|')])) formatted_names = [] for name in names_array: if len(name.strip().split(' ')) == 1: formatted_names.append(name.strip()) else: [firstname, *middlename, lastname] = name.strip().split(' ') if not middlename: formatted_names.append(f'{lastname}, {firstname}') else: formatted_names.append(f'{lastname}, {firstname} {" ".join(middlename)}') piped_names = '|'.join(formatted_names) return piped_names def format_piped_values(values): if is_empty_value(values): return '' # The below ensures any extra pipes/whitespace are removed from piped values (e.g. "x || y " becomes "x|y") return "|".join(list(filter(lambda x: x != "", [x.strip() for x in values.split('|')]))) def format_participant_names(names): # Any blank primary artists would have been excluded in the earlier validation steps # Featuring artists can legitimately be blank if is_empty_value(names): return '' piped_names = re.sub("(, &)|(,)", '|', str(names)) formatted_names = format_piped_values(piped_names) return formatted_names def format_explicit_value(value): if value == '' or type(value) == float and math.isnan(float(value)): return value elif value.lower().strip() == "explicit": return "Yes" elif value.lower().strip() == "clean": return "No" else: return math.nan def format_dataframe(ingest_template_df): ingest_template_df = ingest_template_df.assign(**{ 'Product Primary Artists': ingest_template_df.loc[:,'Product Primary Artists'].apply(lambda x: format_participant_names(x)), 'Product Featuring Artists': ingest_template_df.loc[:,'Product Featuring Artists'].apply(lambda x: format_participant_names(x)), 'Primary Artist': ingest_template_df.loc[:,'Primary Artist'].apply(lambda x: format_participant_names(x)), 'Featuring Artist': ingest_template_df.loc[:,'Featuring Artist'].apply(lambda x: format_participant_names(x)), 'Product Type': 'Audio', 'Sales Start Date': format_date(ingest_template_df.loc[:,'Sales Start Date']), 'Collection Ownership Start Date': format_date(ingest_template_df.loc[:,'Collection Ownership Start Date']), 'Collection Ownership End Date': format_date(ingest_template_df.loc[:,'Collection Ownership End Date']), 'Duration': ingest_template_df.loc[:,'Duration'].apply(lambda x: format_duration(x)), 'Disc Number': ingest_template_df.loc[:,'Disc Number'].fillna('1'), 'Track Number': ingest_template_df.loc[:,'Track Number'].fillna('1'), 'Contributor Legal Name': ingest_template_df.loc[:,'Contributor Legal Name'].apply(lambda x: format_contributor_names(x)), 'Contributor Type': ingest_template_df.loc[:,'Contributor Type'].apply(lambda x: format_contributor_types(x)), 'Contributor Role(s)': ingest_template_df.loc[:,'Contributor Role(s)'].apply(lambda x: format_piped_values(x)), 'Collection Ownership Percentage of Rights': ingest_template_df.loc[:,'Collection Ownership Percentage of Rights'].fillna(100), 'Collection Ownership Country': ingest_template_df.loc[:,'Collection Ownership Country'].apply(lambda x: format_piped_values(x)), 'Collection Ownership Country - Exclusion': ingest_template_df.loc[:, 'Collection Ownership Country - Exclusion'].apply(lambda x: format_piped_values(x)).replace(knr_consts.NO_TERRITORY, ''), 'Country of Label/Individual/Party Which Funded Recording': ingest_template_df.loc[:,'Country of Label/Individual/Party Which Funded Recording'].apply(lambda x: format_piped_values(x)), 'Country of Recording': ingest_template_df.loc[:,'Country of Recording'].apply(lambda x: format_piped_values(x)), 'Explicit': ingest_template_df.loc[:,'Explicit'].apply(lambda x: format_explicit_value(x)) }) invalid_rights_start_dates = ingest_template_df.loc[pd.to_datetime(ingest_template_df.loc[:,'Collection Ownership Start Date'], errors="coerce").isna()] for i, row in invalid_rights_start_dates.iterrows(): release_yr = ingest_template_df.at[i, 'First Year of Release'] ingest_template_df.at[i, 'Collection Ownership Start Date'] = f'01-Jan-{release_yr}' ingest_template_df = strip_all_strings(ingest_template_df) return ingest_template_df def format_video_dataframe(ingest_template_df): ingest_template_df = ingest_template_df.assign(**{ 'Product Primary Artists': ingest_template_df.loc[:,'Product Primary Artists'].apply(lambda x: format_participant_names(x)), 'Product Featuring Artists': ingest_template_df.loc[:,'Product Featuring Artists'].apply(lambda x: format_participant_names(x)), 'Primary Artist': ingest_template_df.loc[:,'Primary Artist'].apply(lambda x: format_participant_names(x)), 'Featuring Artist': ingest_template_df.loc[:,'Featuring Artist'].apply(lambda x: format_participant_names(x)), 'Product Type': 'Video', 'Sales Start Date': format_video_date(ingest_template_df.loc[:,'Sales Start Date']), 'Collection Ownership Start Date': format_date(ingest_template_df.loc[:,'Collection Ownership Start Date']), 'Collection Ownership End Date': format_date(ingest_template_df.loc[:,'Collection Ownership End Date']), 'Duration': ingest_template_df.loc[:,'Duration'].apply(lambda x: format_duration(x)), 'Contributor Legal Name': ingest_template_df.loc[:,'Contributor Legal Name'].apply(lambda x: format_contributor_names(x)), 'Contributor Type': ingest_template_df.loc[:,'Contributor Type'].apply(lambda x: format_contributor_types(x)), 'Contributor Role(s)': ingest_template_df.loc[:,'Contributor Role(s)'].apply(lambda x: format_piped_values(x)), 'Collection Ownership Percentage of Rights': ingest_template_df.loc[:,'Collection Ownership Percentage of Rights'].fillna(100), 'Collection Ownership Country': ingest_template_df.loc[:,'Collection Ownership Country'].apply(lambda x: format_piped_values(x)), 'Collection Ownership Country - Exclusion': ingest_template_df.loc[:, 'Collection Ownership Country - Exclusion'].apply(lambda x: format_piped_values(x)).replace(knr_consts.NO_TERRITORY, ''), 'Country of Label/Individual/Party Which Funded Recording': ingest_template_df.loc[:,'Country of Label/Individual/Party Which Funded Recording'].apply(lambda x: format_piped_values(x)), 'Country of Recording': ingest_template_df.loc[:,'Country of Recording'].apply(lambda x: format_piped_values(x)), 'Explicit': ingest_template_df.loc[:,'Explicit'].apply(lambda x: format_explicit_value(x)) }) invalid_rights_start_dates = ingest_template_df.loc[pd.to_datetime(ingest_template_df.loc[:,'Collection Ownership Start Date'], errors="coerce").isna()] for i, row in invalid_rights_start_dates.iterrows(): release_yr = ingest_template_df.at[i, 'First Year of Release'] ingest_template_df.at[i, 'Collection Ownership Start Date'] = f'01-Jan-{release_yr}' ingest_template_df = strip_all_strings(ingest_template_df) return ingest_template_df def format_date_for_error_file(col): return pd.to_datetime(col, dayfirst=True, errors="ignore").dt.date