"""Beatport Data Ingestion Workflow.""" from datetime import datetime from tempfile import NamedTemporaryFile import boto3 from boto3.s3.transfer import TransferConfig from botocore.exceptions import ClientError from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows.beatport import config from feed_ingestion.flows.beatport.beatport_api import BeatportAPI from feed_ingestion.flows.helpers import get_secret from feed_ingestion.tasks import bootstrap as reload, check_status # Load SQL templates sql_loader = SQLLoader(__file__) @task.decorate(timeout=1000) @reload.reset_dynamodb_status_on_reload(config.feed_name) def bootstrap(activity, date, dw_config=None): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). Returns: dict: Context. """ date_obj = (datetime.strptime( date, '%Y-%m-%d') if date else datetime.today()) activity.logger.info('Bootstrap flow: {}'.format(date_obj)) archive_path = config.s3['archive'].format(date=date_obj) source_file_name = config.filename.format(date=date_obj) temp_staging_raw_table = \ config.temp_staging_raw_table.format(date.replace('-', '_')) s3_archive_path = f's3://{config.data_bucket}/{archive_path}' key_dir = '{}{}'.format(s3_archive_path, source_file_name) return dict( feed_name=config.feed_name, date=date_obj.strftime('%Y-%m-%d'), archive_path=archive_path, source_file_name=source_file_name, s3_archive_path=s3_archive_path, key_dir=key_dir, secrets_path=config.secrets_path, staging_raw_table=config.staging_raw_table, temp_staging_raw_table=temp_staging_raw_table ) @task.decorate(timeout=3600) @check_status() def grab_drop_files( activity, feed_name, date, source_file_name, archive_path): """Copy a feed files from the drop location into the archive folder. Args: activity (ActivityWorker): The activity worker. feed_name (str): The name of the feed. date (str): Reporting date (YYYY-MM-DD). source_file_name (str): The name of the raw file. archive_path (str): s3 path to drop raw files. """ beatport_api = BeatportAPI( supplier_id=config.supplierid, client_id=get_secret(config.secrets_path, 'BEATPORT_CLIENT_ID'), client_secret=get_secret( config.secrets_path, 'BEATPORT_CLIENT_SECRET'), username=get_secret(config.secrets_path, 'BEATPORT_CLIENT_USERNAME'), password=get_secret(config.secrets_path, 'BEATPORT_CLIENT_PASSWORD')) s3 = boto3.client('s3') with NamedTemporaryFile('wb') as file: activity.logger.info( 'Downloading file from Store - {}'.format(file.name)) beatport_api.download_file(file, date) try: activity.logger.info( 'Start uploading file - {}{}'.format( archive_path, source_file_name)) s3.upload_file( file.name, config.data_bucket, f'{archive_path}{source_file_name}', Config=TransferConfig()) except ClientError as e: activity.logger.error( f'Uploading on s3 failed {date} on {source_file_name}') garcon_feed_status.set_missing_files( feed_name, date, [source_file_name]) garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE) raise e