import pandas as pd import os from config import config import sentry_sdk import string from constants import knr_consts from knr_ownership_migration import xls_to_dataframe from utils.validation import validate_contributor_count, valid_value, is_empty_value log = config.get_logger() sentry_sdk.init(config.SENTRY_DSN) def main(env): file_name = os.environ.get("FILE_NAME") target_column = os.environ.get("COLUMN") df = xls_to_dataframe(file_name) df = df[knr_consts.CORRESPONDING_KNR_COLUMNS] df.columns = knr_consts.OWNERSHIP_INGEST_COLUMNS original_length = len(df.index) df = remove_and_log_invalid_contributors(df, env, file_name, target_column) log.info(f'{original_length - len(df.index)} invalid rows of {original_length} original rows.') def remove_and_log_invalid_contributors(df: pd.DataFrame, env, file_name, target_column): try: log.info("Getting invalid contributors") # Dataframe that will hold error rows error_df = pd.DataFrame(columns=knr_consts.OWNERSHIP_INGEST_COLUMNS) # Test each rows' target column, and add any invalid rows to the error_df if target_column == 'Contributor Legal Name': invalid_performers = df.loc[df[target_column].apply(lambda x: invalid_individual_performers_column(x))] invalid_performers.insert(0, "Error", "Invalid Contributor Legal Name") elif target_column == 'Contributor Type': invalid_performers = df.loc[df.loc[:,target_column].apply(lambda x: not valid_value(x, knr_consts.VALID_PERFORMER_CATEGORIES, "|", False, "Contributor Type"))] invalid_performers.insert(0, "Error", "Invalid Contributor Type") elif target_column == 'Contributor Role(s)': invalid_performers = df.loc[df[target_column].apply(lambda x: invalid_performer_roles_column(x))] invalid_performers.insert(0, "Error", "Invalid Contributor Role(s)") # Test the counts of contributors across names, types and roles columns elif target_column == 'Contributor Count': invalid_performers = validate_contributor_count(df) invalid_performers.insert(0, "Error", "Number of Contributor Legal Names, Types, and Roles must be the same.") # Test validity of contributor names and types, as well as counts across names, types and roles columns elif target_column == 'All': invalid_names = df.loc[df['Contributor Legal Name'].apply(lambda x: invalid_individual_performers_column(x))] invalid_names.insert(0, "Error", "Invalid Contributor Legal Name") invalid_types = df.loc[df.loc[:,'Contributor Type'].apply(lambda x: not valid_value(x, knr_consts.VALID_PERFORMER_CATEGORIES, "|", False, "Contributor Type"))] invalid_types.insert(0, "Error", "Invalid Contributor Type") invalid_roles = df.loc[df['Contributor Role(s)'].apply(lambda x: invalid_performer_roles_column(x))] invalid_roles.insert(0, "Error", "Invalid Contributor Role(s)") invalid_counts = validate_contributor_count(df) invalid_counts.insert(0, "Error", "Different number of Contributor Legal Names, Types and Roles") invalid_performers = pd.concat([invalid_names, invalid_types, invalid_roles, invalid_counts]) if len(invalid_performers.index) == 0: return df error_df = pd.concat([error_df, invalid_performers]) # Set Original Row Number column, adding 4 to each index (so that it maps to original excel row number) error_df.sort_index(inplace=True) error_df.index.name = "Original Row Number" xls_indices = list(map(lambda i: i + 4, error_df.index.tolist())) error_df["Original Row Number"] = xls_indices error_df.set_index("Original Row Number", inplace=True) if target_column == "Contributor Legal Name" or target_column == "Contributor Type" or target_column == 'Contributor Role(s)': error_df_filtered = error_df[['Error', 'ISRC', target_column]] write_error_file(error_df_filtered, target_column, file_name) if target_column == "Contributor Count" or target_column == "All": error_df_filtered = error_df[['Error', 'ISRC', "Contributor Legal Name", "Contributor Type", "Contributor Role(s)"]] write_error_file(error_df_filtered, target_column, file_name) # Drop invalid indexes from df, then reset index df.drop(invalid_performers.index, inplace=True, errors="ignore") df.reset_index(drop=True, inplace=True) return df # Catch any exceptions, log, and write error row to csv except Exception as e: log.error(e) pass def write_error_file(file, target_column, file_name): # Only write if errors if(len(file.index >= 1)): file.to_csv(f'data/output/invalid-contributors/invalid-{("".join(e for e in target_column if e.isalnum())).lower()}-{file_name.split(".xls")[0]}.csv') def invalid_individual_performers_column(value): if is_empty_value(value): return False arr = str(value).split("|") invalid_strs = [] for item in arr: x = item.upper().translate(str.maketrans('', '', f'{string.punctuation}')).strip() if item.strip() in knr_consts.KNR_ROLES or x.replace(" ", "") in knr_consts.VALID_PERFORMER_CATEGORIES: invalid_strs.append(x) # remove all empty strings invalid_strs = list(filter(lambda x: x != "", invalid_strs)) return len(invalid_strs) >= 1 def invalid_performer_roles_column(value): if is_empty_value(value): return False arr = str(value).split("|") invalid_strs = [] for item in arr: # Additional space after string.punctuation as VALID_PERFORMER_CATEGORIES have no spaces x = item.upper().translate(str.maketrans('', '', f'{string.punctuation} ')).strip() if x in knr_consts.VALID_PERFORMER_CATEGORIES: invalid_strs.append(x) # remove all empty strings invalid_strs = list(filter(lambda x: x != "", invalid_strs)) return len(invalid_strs) >= 1 if __name__ == "__main__": env = os.environ.get("ENV", 'qa') try: main(env) except Exception as e: log.error(e)