"""Methods for processing data from ripper_feeder.""" import os from PIL import Image from integration_scripts import logger from integration_scripts.file_utils import check_free_space from constants import paths as asset_paths from constants import asset_types as type_const from integration_scripts.file_utils import copy_file, get_s3_file_key import config def check_for_stopfile(tablenum): """Checks for the existence of a stopfile. Args: tablenum: (int) The number of the table for which to check if there exists a stopfile. Returns: (bool) True if the matching stopfile exists; False, otherwise. """ if tablenum: # Check if the process has been stopped previously stop_file = os.path.join( asset_paths.target, asset_paths.log_path, 'asset_ingest_{}.stop'.format(tablenum)) if os.path.isfile(stop_file): return True return False def check_terminal_conditions(tablenum): """Determine if the process is allowed to continue.""" end_process = False msg = '' # Terminate processing if disk space is lower than half the min. free_space = check_free_space(asset_paths.target, config.MIN_FREE_SPACE_GB / 2) if not free_space: msg = 'Not enough free space to continue.' end_process = True has_stopfile = check_for_stopfile(tablenum) # Check stopfile if we're doing automated runs via commandline args if has_stopfile: msg = 'Processing stopped by stopfile. Remove ' \ 'asset_ingest_{}.stop from log folder to resume ' \ 'processing.'.format(tablenum) end_process = True # *** DISABLED - Too risky/potentially slow # # Turn off if email control received. # process_control_email(tablenum=) return end_process, msg def copy_from_local(src_path, target_path): """Copy file from local tmp folder to ripper path.""" # Ensure asset target exists os.makedirs(target_path, exist_ok=True) # This copy_file() uses a higher buffer for speed copy_file(src=src_path, dst=target_path, preserve_file_date=False) def create_donefile_keys(source_relid, orch_upc, use_local): """Create donefile keys.""" # create .done files object keys if use_local: donefile_key = None donefile_backup_key = None release_folder_key = os.path.join( asset_paths.source_audio, source_relid + '/') orch_donefile_key = os.path.join( asset_paths.target, asset_paths.ripper_wav_folder, str(orch_upc), asset_paths.orch_donefile_name(orch_upc)) else: donefile_key = get_s3_file_key( asset_paths.source_audio, source_relid, asset_paths.donefile_name) donefile_backup_key = get_s3_file_key( asset_paths.source_audio, asset_paths.completed, source_relid, asset_paths.donefile_name) # Release-level folder release_folder_key = get_s3_file_key( asset_paths.source_audio, source_relid + '/') # .done for ripper orch_donefile_key = get_s3_file_key( asset_paths.target, asset_paths.ripper_wav_folder, str(orch_upc), asset_paths.orch_donefile_name(orch_upc)) return donefile_backup_key, donefile_key, orch_donefile_key, \ release_folder_key def save_image(asset_target_file, image): """Save image.""" do_continue = False err = None try: # if the image is not square if image.width != image.height: # Check how much longer the long side is long_edge = max(image.width, image.height) short_edge = min(image.width, image.height) percent_larger = (abs(short_edge - long_edge) / short_edge) * 100 thresh = config.FORCE_SQUARE_THRESHOLD force_square = config.FORCE_SQUARE # Reshape if necessary and within threshold if force_square and percent_larger <= thresh: image = image.resize((long_edge, long_edge)) else: raise RuntimeError( 'Image {} is not square, and has a long edge over {}% ' 'longer than the short edge. Will not auto-resize.'.format( asset_target_file, thresh)) if image.mode != 'RGB': logger.info('Converting {} to RGB.', asset_target_file) image = image.convert('RGB') # Upscale dimensions if image.width < 3000 and config.FORCE_UPSCALE: logger.info('Upscaling image {} to 3000x3000px.', asset_target_file) image = image.resize((3000, 3000)) logger.info('Saving File (at 300dpi): {}', asset_target_file) # Save as new file (extension determines type) image.save(asset_target_file, dpi=(300, 300)) except Exception as e: # Open and convert fails # Update action for log action = 'failed' err = e do_continue = True # next result else: # Update action for log action = 'converted' return action, do_continue, err def open_image(tmp_target_path): """Open an image file for processing.""" do_continue = False action = 'opened' image = None err = None # Open File try: image = Image.open(tmp_target_path) except Exception as e: # Open and convert fails # Update action for log action = 'failed' err = e do_continue = True # next result return image, do_continue, action, err # TODO: Change to use loguru file logger from integration_scripts def start_log(table_name, log_date, path, asset_type): """Start the log.""" if asset_type not in type_const.acceptable_types: raise ValueError( '\'type\' must be one of [{}]'.format( ', '.join(type_const.acceptable_types))) log_header = '' log_file_name = '' # Generate log file name and header if asset_type == 'audio': log_header = 'input_table, source_relid, orch_upc, asset_type, ' \ 'release_track_count, completed, skipped, failed, ' \ 'missing, file_count_mismatch, other, restored, ' \ 'timestamp\n' log_file_name = 'bulkassetingest_audio_{}-{}_log.csv'.format( table_name, log_date) elif asset_type == 'cover': log_header = 'input_table, source_relid, orch_upc, asset_type, ' \ 'release_track_count, completed, skipped, failed, ' \ 'missing, file_count_mismatch, other, restored, ' \ 'timestamp\n' log_file_name = 'bulkassetingest_covers_{}-{}_log.csv'.format( table_name, log_date) # Create log file name log_file_name = os.path.join(path, log_file_name) # Start log with open(log_file_name, 'w') as f: f.write(log_header) return log_file_name