""" Apple podcast sales summary monthly Garcon tasks. Tasks to ingest Apple podcast sales summary monthly data. """ from datetime import datetime from datetime import timedelta from dateutil.relativedelta import relativedelta from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.apple_podcasts_sales_summary_monthly import config from feed_ingestion.flows.apple_podcasts_sales_summary_monthly.\ snowflake_executor import ApplePodcastsSalesSummaryMonthlySF from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import check_status @task.decorate(timeout=1000) def bootstrap(activity, date, reload, source, vendor_name, **kwargs): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' delete all feed statuses in DynamoDB. source (str): Source of the report like Daily, Weekly and Monthly. vendor_name (str): Vendor Name(e.g. SME, POD_SUB_LLC) Returns: dict: Initial context for the workflow. """ date_obj = datetime.strptime(date, config.date_format).date() overall_feed_name = config.overall_feed_name activity.logger.info( 'Bootstrap flow: {} for source: {} and vendor_name: {}'.format( date_obj, source, vendor_name)) if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(overall_feed_name, date)) garcon_feed_status.delete_status(overall_feed_name, date) else: overall_status = garcon_feed_status.get_overall_status( overall_feed_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: error_msg = 'Already ingested for flow: {}'.format(date) activity.logger.info(error_msg) return { 'stop': True, 'error_message': error_msg } if source not in \ [config.source_daily, config.source_weekly, config.source_monthly]: error_msg = 'Invalid source: {} for flow: {}'.format(source, date) activity.logger.info(error_msg) return { 'stop': True, 'error_message': error_msg } if source == config.source_monthly: month_end_date = (date_obj + relativedelta(day=31)) if month_end_date != date_obj: error_msg = 'Invalid month end date: {}. It must be: {}'.format( date_obj, month_end_date) activity.logger.info(error_msg) return { 'stop': True, 'error_message': error_msg } if vendor_name not in [config.sme_vendor, config.pod_sub_llc_vendor]: activity.logger.info( 'Invalid vendor_name: {} for flow: {}'.format(vendor_name, date)) return {'stop': True} if source == config.source_daily: expected_file_name = config.daily_report_name.format( vendor_id=config.vendors[vendor_name], date=datetime.strftime(date_obj, config.file_date_format) ) else: expected_file_name = config.report_name.format( source=source, vendor_id=config.vendors[vendor_name], date=datetime.strftime(date_obj, config.file_date_format) ) source_files_dict = {'files': [{'file_name': expected_file_name}]} return { 'date': date_obj.strftime(config.date_format), 'feed_name': overall_feed_name, 'expected_file_name': expected_file_name, 'source_files_dict': source_files_dict, 'secrets_path': config.secrets_path, 's3_archive_path': config.s3_archive_path_template, 'staging_raw_table': config.staging_raw_table, 'source': source, 'vendor_id': config.vendors[vendor_name], 'vendor_name': vendor_name } @task.decorate(timeout=1000) @check_status(feed_name=config.overall_feed_name) def load_sales_summary_monthly_table( activity, date, source, staging_raw_table, vendor_id, vendor_name): """Load sales summary monthly table. Also, clear staging raw sales summary monthly table. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): Source of the report like Daily, Weekly and Monthly. staging_raw_table (str): A staging table name in Snowflake. vendor_id (str): Vendor ID. vendor_name (str): Vendor Name(e.g. SME, POD_SUB_LLC). """ with ApplePodcastsSalesSummaryMonthlySF( get_sf_config(config.secrets_path)) as executor: activity.logger.info('Load sales summary monthly table') executor.load_sales_summary_monthly_table( staging_raw_table, date, source, vendor_id, vendor_name) activity.logger.info('Clear staging raw sales summary monthly table') executor.delete_from_staging_raw_sales_summary_monthly( staging_raw_table, date, source, vendor_id) @task.decorate(timeout=1000) @check_status(feed_name=config.overall_feed_name) def delete_data_for_old_source(activity, date, source, vendor_id): """Delete entries which are from old source. For eg: delete daily records for that week after weekly ingestion, delete daily and weekly records for that month after monthly ingestion. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): Source of the report like Daily, Weekly and Monthly. vendor_id (str): Vendor ID. """ with ApplePodcastsSalesSummaryMonthlySF( get_sf_config(config.secrets_path)) as executor: download_date = datetime.strptime(date, config.date_format) if source == config.source_weekly: delete_sources = (config.source_daily,) from_date = ( download_date - timedelta(days=6) ).strftime(config.date_format) to_date = date elif source == config.source_monthly: delete_sources = (config.source_daily, config.source_weekly) from_date = f'{download_date.year}-{download_date.month}-01' to_date = date activity.logger.info( f'Delete entries for old source from sales summary monthly table.' f'\nFor sources: {delete_sources} and from_date: {from_date}' f' and to_date: {to_date}' ) executor.delete_data_for_old_source( from_date, to_date, delete_sources, vendor_id)