"""This script will check that the list of tracks provided (UPC_VOL_TRK.wav) from a text file matches the list of files on an asset path.""" import os from integration_scripts import file_utils as fu from integration_scripts import logger # Define vars file_name = '/Users/lvona/Documents/python_projects/ripper-feeder/' \ 'Suburban_2020_ingestion_list.dat' root_path = '/Volumes/ripped_assets/ripper_feeder_staging/Suburban/' \ 'ready_to_copy/2020_01_14' def main(): # Get list of files from root path. with open(file_name) as f: metadata_processed = f.readlines() metadata_processed = [line.rstrip('\n') for line in metadata_processed] metadata_processed.sort() # Get list of files from text file. data_source_assets = fu.get_all_files_in_subdirs(root_path, '*.wav') data_source_assets.sort() # Report list lengths logger.info('Metadata reports {} assets.', len(metadata_processed)) logger.info('Data Source reports {} assets.', len(data_source_assets)) # Compare Lists if metadata_processed != data_source_assets: # -- DUMB CHECKS ------------------------------------------------ # Check for dupes in each list metadata_set = set(metadata_processed) data_source_set = set(data_source_assets) # Report list lengths logger.info('Metadata reports {} unique assets.', len(metadata_set)) logger.info('Data Source reports {} unique assets.', len(data_source_set)) if metadata_set == data_source_set: logger.info('File lists match.') return if len(metadata_set) < len(metadata_processed): logger.warning('There are duplicates in the METADATA list.') if len(data_source_set) < len(data_source_assets): logger.warning('There are duplicates in the DATA SOURCE list.') # Get list lengths metadata_set_len = len(metadata_set) prepped_set_len = len(data_source_set) if metadata_set_len > prepped_set_len: bigger = 'METADATA has more files.' elif metadata_set_len < prepped_set_len: bigger = 'DATA SOURCE has more files.' else: bigger = 'Lists are the same size, but differ.' # Report to User logger.warning('Files Do No Match. {}', bigger) # -- FILTERED CHECKS -------------------------------------------- diff_set_1 = list(set([x for x in metadata_set if x not in data_source_set])) diff_set_1.sort() diff_set_2 = list(set([x for x in data_source_set if x not in metadata_set])) diff_set_2.sort() diff_set = [*diff_set_1, *diff_set_2] diff_set.sort() intersection = [x for x in metadata_set if x in data_source_set] intersection_upcs = list(set([ li.split('_')[0] + '\n' for li in intersection])) intersection_upcs.sort() # Check for different values # In Metadata, Not in Data Source not_in_data_source = [ x for x in metadata_set if x not in data_source_set] not_in_data_source.sort() # In Data Source, Not in Metadata not_in_metadata = [ x for x in data_source_set if x not in metadata_set] not_in_metadata.sort() # UPC's where in Metadata, Not in Data Source not_in_data_source_upc_list = list(set([ li.split('_')[0] + '\n' for li in not_in_data_source])) not_in_data_source_upc_list.sort() # UPC's where in Data Source, Not in Metadata not_in_metadata_upc_list = list(set([ li.split('_')[0] + '\n' for li in not_in_metadata])) not_in_metadata_upc_list.sort() # Len of upc where not in Metadata not_in_metadata_upc_list_len = len(not_in_metadata_upc_list) # Len of upc where not in Data Source not_in_data_source_upc_list_len = len(not_in_data_source_upc_list) logger.warning( 'UPC\'s with files not in METADATA set - {} UPC\'s:', not_in_metadata_upc_list_len) for line in not_in_metadata_upc_list: logger.warning(line.rstrip('\n')) logger.warning( 'UPC\'s with files not in DATA SOURCE set - {} UPC\'s:', not_in_data_source_upc_list_len) for line in not_in_data_source_upc_list: logger.warning(line.rstrip('\n')) # Name output file output_file = os.path.splitext(file_name)[0] + '.txt' logger.warning('Writing diff file {}', output_file) # Full Diff output_list = [li + '\n' for li in list(diff_set)] output_list.sort() not_in_data_source = [li + '\n' for li in list(not_in_data_source)] not_in_data_source.sort() not_in_metadata = [li + '\n' for li in list(not_in_metadata)] not_in_metadata.sort() not_in_data_source_upc_list = [ li for li in list(not_in_data_source_upc_list)] not_in_data_source_upc_list.sort() not_in_metadata_upc_list = [ li for li in list(not_in_metadata_upc_list)] not_in_metadata_upc_list.sort() intersection = [li + '\n' for li in list(intersection)] intersection.sort() # Print track list # for line in output_list: # logger.warning(line.rstrip('\n')) # Get upc list upc_list = set([li.split('_')[0] + '\n' for li in output_list]) # logger.warning('UPC List') # Print upc list # for line in upc_list: # logger.warning(line.rstrip('\n')) with open(output_file, 'w') as of: of.write( 'UPC\'s with all files found in both sources - ' '{} UPC\'s\n'.format(len(intersection_upcs))) of.writelines(intersection_upcs) of.write( '\n\nFiles found in both sources - {} Tracks\n'.format(len( intersection))) of.writelines(intersection) of.write('\n\nNot In Data Source - UPC\'s - {} UPC\'s\n'.format( not_in_data_source_upc_list_len)) of.writelines(not_in_data_source_upc_list) of.write( '\n\nNot In Metadata - UPC\'s - {} UPC\'s\n'.format( not_in_metadata_upc_list_len)) of.writelines(not_in_metadata_upc_list) of.write( '\n\nNot In Data Source - Track List - {} Tracks\n'.format( len(not_in_data_source))) of.writelines(not_in_data_source) of.write( '\n\nNot In Metadata - Track List - {} Tracks\n'.format( len(not_in_metadata))) of.writelines(not_in_metadata) of.write('\n\nTOTAL DIFF - UPC List - {} UPC\'s\n'.format( len(upc_list))) of.writelines(upc_list) if __name__ == '__main__': main()