""" Apple podcast subscription snapshot monthly Garcon tasks. Tasks to ingest Apple podcast subscription snapshot monthly data. """ from datetime import datetime from datetime import timedelta from dateutil.relativedelta import relativedelta from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.apple_podcasts_subscription_snapshot_monthly \ import config from feed_ingestion.flows.apple_podcasts_subscription_snapshot_monthly \ .snowflake_executor import ApplePodcastsSubsSnapshotMonthlySF from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import check_status @task.decorate(timeout=1000) def bootstrap(activity, date, reload, source, vendor_name, **kwargs): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): source of the report like Daily, Weekly and Monthly. reload (str or None): If 'True' delete all feed statuses in DynamoDB. vendor_name (str): Vendor Name(e.g. POD_SUB_LLC, SME). Returns: dict: Initial context for the workflow. """ date_obj = datetime.strptime(date, config.date_format).date() activity.logger.info( 'Bootstrap flow: {} for source: {} and vendor_name: {}'.format( date_obj, source, vendor_name)) final_feed_name = '_'.join([ config.overall_feed_name, source, vendor_name]) activity.logger.info(f'Feed Name: {final_feed_name}') if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(final_feed_name, date)) garcon_feed_status.delete_status(final_feed_name, date) else: overall_status = garcon_feed_status.get_overall_status( final_feed_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: error_msg = 'Already ingested for flow: {} for date: {}'.format( final_feed_name, date) return {'stop': True, 'error_message': error_msg} if source not in config.sources: error_msg = f'Invalid source: {source} for flow: {date}' activity.logger.info(error_msg) return {'stop': True, 'error_message': error_msg} if source == config.source_monthly: month_end_date = (date_obj + relativedelta(day=31)) if month_end_date != date_obj: error_msg = 'Invalid month end date: {}. It must be: {}'.format( date_obj, month_end_date) activity.logger.info(error_msg) return {'stop': True, 'error_message': error_msg} if vendor_name not in [config.sme_vendor, config.pod_sub_llc_vendor]: error_msg = f'Invalid vendor_name: {vendor_name} for flow: {date}' activity.logger.info(error_msg) return {'stop': True, 'error_message': error_msg} expected_file_name = config.report_name.format( vendor_id=config.vendors[vendor_name], date=datetime.strftime(date_obj, config.file_date_format) ) source_files_dict = {'files': [{'file_name': expected_file_name}]} return { 'date': date_obj.strftime(config.date_format), 'feed_name': final_feed_name, 'expected_file_name': expected_file_name, 'source_files_dict': source_files_dict, 'secrets_path': config.secrets_path, 's3_archive_path': config.s3_archive_path_template, 'staging_raw_table': config.staging_raw_table, 'source': source, 'vendor_id': config.vendors[vendor_name], 'vendor_name': vendor_name } @task.decorate(timeout=1000) @check_status() def load_subscription_snapshot_monthly_table( activity, feed_name, date, source, staging_raw_table, vendor_id, vendor_name): """Load subscription snapshot monthly table. Also, clear staging raw subscription snapshot monthly table. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): source of the report like Daily, Weekly and Monthly. staging_raw_table (str): A staging table name in Snowflake. vendor_id (str): vendor id. vendor_name (str): Vendor Name(e.g. POD_SUB_LLC, SME) """ with ApplePodcastsSubsSnapshotMonthlySF( get_sf_config(config.secrets_path)) as executor: activity.logger.info('Load subscription snapshot monthly table') executor.load_subscription_snapshot_monthly_table( staging_raw_table, date, source, vendor_id, vendor_name) activity.logger.info( 'Clear staging raw subscription snapshot monthly table') executor.delete_from_staging_raw_subscription_snapshot_monthly( staging_raw_table, date, source, vendor_id) @task.decorate(timeout=1000) @check_status() def delete_data_for_old_source(activity, feed_name, date, source, vendor_id): """Delete entries which are from old source. For eg: delete daily records for that week after weekly ingestion, delete daily and weekly records for that month after monthly ingestion. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): source of the report like Daily, Weekly and Monthly. vendor_id (str): vendor id. """ with ApplePodcastsSubsSnapshotMonthlySF( get_sf_config(config.secrets_path)) as executor: download_date = datetime.strptime(date, config.date_format) if source == config.source_weekly: delete_sources = (config.source_daily,) from_date = ( download_date - timedelta(days=6) ).strftime(config.date_format) to_date = date elif source == config.source_monthly: delete_sources = (config.source_daily, config.source_weekly) from_date = f'{download_date.year}-{download_date.month}-01' to_date = date activity.logger.info( f'Delete entries for old source from snapshot monthly table.\n' f'For sources: {delete_sources} and from_date: {from_date}' f' and to_date: {to_date}' ) executor.delete_data_for_old_source( from_date, to_date, delete_sources, vendor_id)