""" Apple podcast subscription events Garcon tasks. Tasks to ingest Apple podcast subscription events monthly data. """ from datetime import datetime from datetime import timedelta from dateutil.relativedelta import relativedelta from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows.apple_podcasts_subscription_events_monthly\ import config from feed_ingestion.flows.apple_podcasts_subscription_events_monthly.\ snowflake_executor import ApplePodcastsSubscriptionEventsMonthlySF from feed_ingestion.flows.helpers import get_sf_config from feed_ingestion.tasks import check_status @task.decorate(timeout=1000) def bootstrap(activity, date, reload, source, vendor_name, **kwargs): """Bootstrap workflow by injecting initial context from config. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' delete all feed statuses in DynamoDB. source (str): source of the report like Daily, Weekly and Monthly. Returns: dict: Initial context for the workflow. """ date_obj = datetime.strptime(date, config.date_format).date() activity.logger.info( 'Bootstrap flow: {} for source: {} and vendor_name: {}'.format( date_obj, source, vendor_name)) final_feed_name = '_'.join([ config.overall_feed_name, source, vendor_name]) activity.logger.info(f'Feed Name: {final_feed_name}') if reload == 'True': activity.logger.info( 'Delete status for feed: {} {} '.format(final_feed_name, date)) garcon_feed_status.delete_status(final_feed_name, date) else: overall_status = garcon_feed_status.get_overall_status( final_feed_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: error_msg = 'Already ingested for flow: {} for date: {}'.format( final_feed_name, date) return {'stop': True, 'error_message': error_msg} if source not in \ [config.source_daily, config.source_weekly, config.source_monthly]: error_msg = f'Invalid source: {source} for flow: {date}' activity.logger.info(error_msg) return {'stop': True, 'error_message': error_msg} if source == config.source_monthly: month_end_date = (date_obj + relativedelta(day=31)) if month_end_date != date_obj: error_msg = 'Invalid month end date: {}. It must be: {}'.format( date_obj, month_end_date) activity.logger.info(error_msg) return {'stop': True, 'error_message': error_msg} if vendor_name not in [config.sme_vendor, config.pod_sub_llc_vendor]: error_msg = f'Invalid vendor_name: {vendor_name} for flow: {date}' activity.logger.info(error_msg) return {'stop': True, 'error_message': error_msg} expected_file_name = config.report_name.format( vendor_id=config.vendors[vendor_name], date=datetime.strftime(date_obj, config.file_date_format) ) source_files_dict = {'files': [{'file_name': expected_file_name}]} return { 'date': date_obj.strftime(config.date_format), 'feed_name': final_feed_name, 'expected_file_name': expected_file_name, 'source_files_dict': source_files_dict, 'secrets_path': config.secrets_path, 's3_archive_path': config.s3_archive_path_template, 'staging_raw_table': config.staging_raw_table, 'source': source, 'vendor_id': config.vendors[vendor_name], 'vendor_name': vendor_name } @task.decorate(timeout=1000) @check_status() def load_subscription_events_monthly_table( activity, feed_name, date, source, staging_raw_table, vendor_id, vendor_name): """Load subscription events monthly table delete staging raw table. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): source of the report like Daily, Weekly and Monthly. staging_raw_table (str): A staging table name in Snowflake. """ with ApplePodcastsSubscriptionEventsMonthlySF( get_sf_config(config.secrets_path)) as executor: activity.logger.info('Load subscription events monthly table') executor.load_subscription_events_monthly_table( staging_raw_table, date, source, vendor_id, vendor_name) activity.logger.info('Clear staging raw subs events monthly table') executor.delete_from_staging_raw_subscription_events_monthly( staging_raw_table, date, source, vendor_id) @task.decorate(timeout=1000) @check_status() def delete_data_for_old_source(activity, feed_name, date, source, vendor_id): """Delete entries which are from old source. For eg: delete daily records for that week after weekly ingestion, delete daily and weekly records for that month after monthly ingestion. Args: activity (ActivityWorker): The Garcon activity worker. date (str): Reporting date (YYYY-MM-DD). source (str): source of the report like Daily, Weekly and Monthly. """ with ApplePodcastsSubscriptionEventsMonthlySF( get_sf_config(config.secrets_path)) as executor: download_date = datetime.strptime(date, config.date_format) if source == config.source_weekly: delete_sources = (config.source_daily,) from_date = ( download_date - timedelta(days=6) ).strftime(config.date_format) to_date = date elif source == config.source_monthly: delete_sources = (config.source_daily, config.source_weekly) from_date = f'{download_date.year}-{download_date.month}-01' to_date = date activity.logger.info( f'Delete entries for old source from subs events monthly table.\n' f'For sources: {delete_sources} and from_date: {from_date}' f' and to_date: {to_date}' ) executor.delete_data_for_old_source( from_date, to_date, delete_sources, vendor_id)