"""Tasks of the Apple Id Mapping Sme Ingestion Workflow.""" from collections import defaultdict from datetime import date as date_module from datetime import datetime import boto3 from botocore.exceptions import ClientError as BotoClientError from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.conf.config import BOTO3_CONFIG from feed_ingestion.flows.apple_id_mapping_sme import config from feed_ingestion.flows.apple_id_mapping_sme.snowflake_executor import \ AppleIDMapping from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import bootstrap as reload from feed_ingestion.tasks import check_ingested_status from feed_ingestion.tasks import check_status STOP_RESPONSE = defaultdict(str, {'stop': True}) @task.decorate(timeout=700) @reload.reset_dynamodb_status_on_reload(config.feed_name) @check_ingested_status(config.feed_name) def check_feed_status(activity, date, dw_config=None): """Check and reset feed status if it is needed. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). dw_config (dict): Dictionary of data warehouse config options. Returns: dict: {} or STOP_RESPONSE from check_ingested_status decorator """ return {} @task.decorate(timeout=700) def bootstrap(activity, date, snowflake_error_limit): """Bootstrap workflow by getting the correct configurations. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). snowflake_error_limit (int or None): Snowflake error limit. Returns: dict: Context. """ date = date or date_module.today().strftime('%Y-%m-%d') date_obj = datetime.strptime(date, '%Y-%m-%d').date() activity.logger.info('Bootstrap flow for {}'.format(date)) s3_archive_path = config.s3['archive_path'].format(date=date_obj) reports = {} for report in config.reports: reports[report] = dict( filename=config.file_template.format(report=report, date=date_obj), temp_table=config.temp_table_name.format( report=report, date=date_obj), s3_download_path=config.s3['drop_path'].format( report=report, date=date_obj)) if snowflake_error_limit: if isinstance(snowflake_error_limit, int): snowflake_error_limit = snowflake_error_limit else: return dict( stop=True, error='snowflake_error_limit should be integer') else: # The smallest available value snowflake_error_limit = 1 return dict( feed_name=config.feed_name, date=date, secrets_path=config.secrets_path, s3_archive_path=s3_archive_path, reports=reports, snowflake_error_limit=snowflake_error_limit) @task.decorate(timeout=7000) @check_status() def fetch_from_drop_location( activity, feed_name, date, s3_archive_path, reports): """Archive certain files from the drop location to archive location. Args: activity (ActivityWorker): The activity worker. feed_name (str): Name of feed being ingested. date (str): Reporting date (YYYY-MM-DD). s3_archive_path (str): Archive location on S3. reports (dict): dict containing metadata of files. Returns: dict: Context patch. """ sme_s3_client = _get_sme_s3_client_assume_role(config.role_arn) for report, description in reports.items(): copy_source = { 'Bucket': config.drop_bucket, 'Key': '{s3_path}{filename}'.format( s3_path=description['s3_download_path'], filename=description['filename']) } archive_path = '{s3_path}{filename}'.format( s3_path=s3_archive_path, filename=description['filename']) try: sme_s3_client.get_object( Bucket=copy_source.get('Bucket'), Key=copy_source.get('Key'), )['ContentLength'] except BotoClientError as err: if err.response['Error']['Code'] == 'NoSuchKey': msg = (f"Cannot find {copy_source.get('Key')} object on S3") activity.logger.error(msg) garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_NOT_AVAILABLE) STOP_RESPONSE.update(message=msg) return STOP_RESPONSE raise err sme_s3_client.copy( copy_source, config.archive_bucket, archive_path) activity.logger.info('File {} uploaded'.format( description['filename'])) def _get_sme_s3_client_assume_role(role_arn): """Assume role_arn and return new boto3 Client connection. Args: role_arn (str): The role_arn. Returns: Connection boto3.session.Session.client. """ client = boto3.client('sts', config=BOTO3_CONFIG) response = client.assume_role( RoleArn=role_arn, RoleSessionName='feed_ingestion')['Credentials'] credentials = { 'aws_access_key_id': response['AccessKeyId'], 'aws_secret_access_key': response['SecretAccessKey'], 'aws_session_token': response['SessionToken'] } return boto3.client('s3', **credentials) @task.decorate(timeout=10000) @check_status() def update_apple_id_mapping(activity, date, feed_name, secrets_path=None): """Update apple_id_mapping table with new data. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). feed_name (str): Name of feed being ingested. secrets_path (str): Secrets manager path of the flow. """ sf_config = get_sf_config(config.secrets_path) with AppleIDMapping(sf_config) as executor: executor.update_sony_apple_id_mapping() activity.logger.info( 'sony_apple_id_mapping successfully updated at {}'.format(date))