"""The module contains activities' tasks.""" __all__ = [ 'bootstrap', 'check_feed_status', 'copy_blob_to_archive', ] from datetime import date as date_module, datetime from botocore.exceptions import ClientError from dateutil.relativedelta import relativedelta from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.spotify_artificial_streams import config from feed_ingestion.tasks import check_status from feed_ingestion.tasks import s3_tasks _STOP_RESPONSE = {'stop': True} @task.decorate(timeout=config.default_task_timeout) def check_feed_status(activity, date, reload): """Check and reset feed status if it is needed. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (Optional[str]): If 'True' delete all feed statuses in DynamoDB. Returns: dict: {} or STOP_RESPONSE from check_ingested_status decorator """ workflow_day = date_module.fromisoformat(date).day if workflow_day != config.workflow_month_day: raise ValueError( f'Current workflow month day is {workflow_day}, ' f'but only {config.workflow_month_day} is permitted ' f'not to initiate SCD type 2 process.' ) if reload == 'True': # noqa: F821 activity.logger.info( 'Delete status for feed: {} {} '.format(config.feed_name, date) ) garcon_feed_status.delete_status(config.feed_name, date) else: overall_status = garcon_feed_status.get_overall_status( config.feed_name, date, ) if overall_status == garcon_feed_status.STATUS_INGESTED: return _STOP_RESPONSE return {'feed_name': config.feed_name} @task.decorate(timeout=config.default_task_timeout) def bootstrap( activity, reload, date, ): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' delete all feed statuses in DynamoDB. Returns: dict: Initial context for the workflow. """ current_datetime = datetime.fromisoformat(date) if date else datetime.now() if reload == 'True': garcon_feed_status.delete_status(config.feed_name, date) previous_month_datetime = current_datetime - relativedelta(months=1) first_day_of_the_month_str = '{current_month_id}01'.format( current_month_id=previous_month_datetime.strftime('%Y%m'), ) last_day_of_the_month_str = ( previous_month_datetime + relativedelta(day=31) ).strftime('%Y%m%d') blob_file_name_wildcard = ( f'.*{config.blob_name_prefix}' f'-{first_day_of_the_month_str}' f'-{last_day_of_the_month_str}' f'.*.{config.blob_name_extension}' ) drop_blob_bucket = config.s3_blobs_prefixes['drop_blob_bucket'] drop_blob_path = config.s3_blobs_prefixes['drop_blob_path'] full_drop_blob_names = ( f's3://{drop_blob_bucket}/{drop_blob_path}/{blob_file_name_wildcard}' ) archive_blob_prefix = '{archive_blob_prefix}/{date}'.format( archive_blob_prefix=config.s3_blobs_prefixes['archive_blob_prefix'], date=date ) return { 'archive_blob_prefix': archive_blob_prefix, 'date': current_datetime.date().isoformat(), 'drop_bucket_name': drop_blob_bucket, 'drop_blob_path': drop_blob_path, 'drop_blob_wildcard': blob_file_name_wildcard, 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'staging_raw_load_kwargs': { 'blob_path': full_drop_blob_names, }, 'temp_staging_raw_load_kwargs': { 'file_pattern': blob_file_name_wildcard, 'snowflake_error_limit': config.snowflake_error_limit, }, 'temp_staging_raw_name': config.temp_staging_table_name.format( current_datetime.strftime('%Y%m%d'), ), 'snowflake_error_limit': config.snowflake_error_limit, 'staging_raw_table': config.staging_raw_table_name, } @task.decorate(timeout=36000) @check_status(feed_name=config.feed_name, task_id='copy_blob_to_archive') def copy_blob_to_archive( activity, date, archive_blob_prefix, drop_bucket_name, drop_blob_path, drop_blob_wildcard, reload, ): """Upload files to archive location. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYY-MM-DD). reload (Optional[str]): if 'True' the flow run is reload of data. archive_blob_prefix (str): prefix for blobs to copy to. drop_bucket_name (str): name of the bucket with dropped objects. drop_blob_path (str): path to dropped objects. drop_blob_wildcard (str): dropped objects names wildcard. """ try: if reload == 'True': s3_tasks.remove_files_from_path( activity=activity, path=archive_blob_prefix, return_deleted_files=False, ) files = s3_tasks.source_files( activity=activity, s3_bucket=drop_bucket_name, s3_path=drop_blob_path, file_pattern=drop_blob_wildcard, ) s3_tasks.copy_files( activity=activity, s3_archive_path=archive_blob_prefix, s3_download_path=f's3://{drop_bucket_name}/{drop_blob_path}', source_files_dict=files['source_files_dict'], ) except ClientError as e: activity.logger.info( f'blobs {drop_blob_wildcard} ' f'have not been copied to {archive_blob_prefix} ' f'because of {e}.' ) return _STOP_RESPONSE garcon_feed_status.set_overall_status( config.feed_name, date, garcon_feed_status.STATUS_DOWNLOADED, )