"""Deezer Marketshare Garcon tasks.""" from datetime import datetime import os from garcon import task from garcon_contrib.aws.utils import garcon_s3 from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.deezer_marketshare import config from feed_ingestion.tasks import bootstrap from feed_ingestion.util import os_tools, task_status from feed_ingestion.util.aws import s3 as s3utils import feed_ingestion.util.deezer_zephir_utils as zephir STOP_RESPONSE = {'stop': True} @task.decorate(timeout=1000) @bootstrap.reset_dynamodb_status_on_reload(config.feed_name) def bootstrap(activity, date, reload): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str): Should we reload the flow. Returns: dict: Context. """ date_obj = datetime.strptime(date, '%Y-%m-%d') s3_archive_path = config.s3['archives'].format(date=date_obj) s3_processed_path = config.s3['processed'].format(date=date_obj) zephir_zip_pattern = config.zip_pattern.format(date=date_obj) bootstrap_result = { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': date, 's3_archive_path': s3_archive_path, 's3_temp_staging_raw_bucket': s3_processed_path, 'zephir_path': config.zephir.get('path'), 'zephir_zip_pattern': zephir_zip_pattern, 'drop_file_name': config.summary_file, 'staging_raw_table': config.snowflake_table_names['staging_raw'], } return bootstrap_result @task.decorate(timeout=1000) def fetch_from_drop_location( activity, feed_name, date, source_path, target_s3_path, filename_pattern): """Download a feed file from the drop location into the archive folder. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). feed_name (str): Name of the feed. source_path (str): Remote Zephir route & params to the source file. target_s3_path (str): Destination S3 path to the archive location. filename_pattern (str): The filename pattern to search by. """ task_id = 'fetch_from_drop_location' # Short circuit flow if overall status is already INGESTED if garcon_feed_status.get_overall_status( feed_name, date) == garcon_feed_status.STATUS_INGESTED: activity.logger.info('Feed already ingested for {}'.format(date)) return { 'stop': True, 'message': '{feed_name} is already ingested for {date}'.format( feed_name=feed_name, date=date)} # Short circuit flow if task status is already Completed if task_status.is_completed_task(feed_name, date, task_id): activity.logger.info( 'The files for {date} have been already ' 'copied from Zephir.'.format(date=date)) return # setup Zephir settings zephir_settings = dict( username=config.zephir.get('username'), password=config.zephir.get('password'), host=config.zephir.get('host'), search=config.zephir.get('search'), path=source_path ) try: local_dir = os_tools.create_temp_dir(feed_name=feed_name, date=date) local_files = zephir.download_invoice_from_zephir( zephir_settings, filename_pattern, local_dir) activity.logger.info( '{files} were downloaded from drop location.'.format( files=', '.join(local_files))) except FileNotFoundError as err: garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE) garcon_feed_status.set_missing_files( feed_name, date, [filename_pattern]) activity.logger.error(str(err)) return {'stop': True, 'message': str(err)} s3_bucket_name, s3_dir_key = garcon_s3.extract_bucket_path(target_s3_path) result = [] for file in local_files: filename = file.split('/')[-1] s3_file_key = os.path.join(s3_dir_key, filename) # delete from S3 if exists s3utils.delete_s3_obj(s3_bucket_name, s3_file_key) activity.logger.info("'{}' was succesfully deleted from '{}'".format( s3_file_key, s3_bucket_name)) # Upload downloaded file to s3 file_size = s3utils.upload_to_s3( file, s3_bucket_name, s3_file_key) if file_size is None: garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_NOT_INGESTED) return {'stop': True, 'message': "Upload to '{}' has failed".format(s3_file_key)} activity.logger.info( '{file} ({file_size}) was uploaded to S3 archives.'.format( file_size=file_size, file=file)) # Remove file from local dir when done uploading os.remove(file) result.append({ 'file_name': filename, 'file_size': file_size, 'found': True }) task_status.mark_completed_task(feed_name, date, task_id) return dict(source_files_dict={'files': result})