"""Tasks for the TikTok OpenEscrow Conflict Report Ingestion Workflow.""" from datetime import datetime import os from garcon import task from feed_ingestion.flows.tiktok_openescrow import config from feed_ingestion.tasks import bootstrap as reload from feed_ingestion.util import task_status from feed_ingestion.util.aws import s3 @task.decorate(timeout=1000) @reload.reset_dynamodb_status_on_reload(config.feed_name) def bootstrap(activity, date, reload): """Bootstrap the workflow with the configuration for the reporting month. Args: activity (ActivityWorker): The activity worker. date (str): First day of the reporting month (YYYY-MM-DD). reload (str or None): If 'True', the feed status is reset upstream. Returns: dict: Initial context for the workflow. """ date_obj = ( datetime.strptime(date, '%Y-%m-%d') if date else datetime.today() ) activity.logger.info('Bootstrap flow: {}'.format(date_obj)) # Source (orchard FTP share) - file is dropped as a .zip. drop_path = config.s3['drop'] source_file_name = config.filename.format(date=date_obj, format='zip') s3_drop_file_path = ( f's3://{config.drop_bucket}/{drop_path}/{source_file_name}' ) # Archive - file is converted to .gz before being uploaded for loading. new_file_name = config.filename.format(date=date_obj, format='gz') archive_path = config.s3['archive'].format( feed_name=config.feed_name, date=date_obj ) s3_dir_path = f's3://{config.data_bucket}/{archive_path}/' s3_archive_file_path = f'{s3_dir_path}{new_file_name}' return dict( date=date_obj.strftime('%Y-%m-%d'), feed_name=config.feed_name, drop_path=drop_path, archive_path=archive_path, source_file_name=source_file_name, new_file_name=new_file_name, s3_drop_file_path=s3_drop_file_path, s3_archive_file_path=s3_archive_file_path, s3_dir_path=s3_dir_path, secrets_path=config.secrets_path, staging_raw_table=config.staging_raw_table ) @task.decorate(timeout=3600) def grab_drop_files(activity, date, s3_drop_file_path, s3_archive_file_path): """Archive the dropped .zip report to S3 as .gz. Status is checked inline (not via the check_status decorator) so a resumed run skips the conversion but still returns source_files_dict for the load. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). s3_drop_file_path (str): S3 path of the dropped .zip file. s3_archive_file_path (str): S3 path for the archived .gz file. Returns: dict: source_files_dict for the archived file, or a stop signal if the file has not arrived yet. """ if task_status.is_completed_task( config.feed_name, date, 'grab_drop_files'): activity.logger.info( f'grab_drop_files already completed for {date} - reusing ' f'archived file {s3_archive_file_path}' ) else: activity.logger.info( f'Checking if there are any files in {s3_drop_file_path} to ' f'process - ' f'{s3.get_list_of_files_and_directories(s3_drop_file_path)}' ) if not s3.get_list_of_files_and_directories(s3_drop_file_path): activity.logger.info(f'No files found in {s3_drop_file_path}') return {'stop': True, 'missing_file': s3_drop_file_path} activity.logger.info( f'Convert ZIP {s3_drop_file_path} to {s3_archive_file_path}' ) s3.convert_zip_to_gzip_on_s3( activity, zip_s3_path=s3_drop_file_path, gz_s3_path=s3_archive_file_path, local_temp_dir='./', extract_original_filename=True, ) task_status.mark_completed_task( config.feed_name, date, 'grab_drop_files') # The zip archive holds a single report file, so build a one-entry # source_files_dict pointing at the archived gzip file. return { 'source_files_dict': { 'files': [{ 'file_name': os.path.basename(s3_archive_file_path), 'file_size': s3.get_key_size(s3_archive_file_path), 'found': True, }], }, }