""" Apple podcast subscription snapshot monthly data ingestion workflow. Extract, transform and load Apple podcast subscription snapshot monthly report raw data to apple_podcasts_subscription_snapshot_monthly table. """ from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.apple_podcasts_subscription_snapshot_monthly \ import config from feed_ingestion.flows.apple_podcasts_subscription_snapshot_monthly \ import tasks from feed_ingestion.flows.apple_podcasts_subscription_snapshot_monthly\ .stage_loader import ApplePodcastsSnapshotMonthlyReportsSL from feed_ingestion.flows.base import FlowBase from feed_ingestion.flows.base import FlowConfigMixin from feed_ingestion.tasks import apple_podcasts_reporter_tasks from feed_ingestion.tasks import overall_status_tasks class Flow(FlowBase, FlowConfigMixin): """Class representing a workflow.""" def __init__(self): """Initialize flow object.""" super().__init__(feed_name=config.overall_feed_name, version='2.0') self.timeout = 21600 # override default to 6 hours (60 *60 * 6) def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ assert 'source' in context, 'There is no source in context' assert context['source'] in [ config.source_daily, config.source_weekly, config.source_monthly, ], 'The source {} is invalid in context'.format(context['source']) assert 'vendor_name' in context, 'There is no vendor_name in context' assert context['vendor_name'] in [ config.sme_vendor, config.pod_sub_llc_vendor, ], 'The vendor_name {} is invalid in context'.format( context['vendor_name'] ) return '_'.join([ self.feed_name, context['source'], context['vendor_name'], ]) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return reporter_to_s3 = schedule( 'reporter_to_s3', self.reporter_to_s3, requires=[bootstrap]) if reporter_to_s3.result.get('reporter_to_s3.file_name')\ != bootstrap.result.get('bootstrap.expected_file_name') or \ (reporter_to_s3.result.get('reporter_to_s3.file_name') == bootstrap.result.get('bootstrap.expected_file_name') and reporter_to_s3.result.get('reporter_to_s3.status') != garcon_feed_status.STATUS_DOWNLOADED): return set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[reporter_to_s3]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[set_status_to_downloaded]) set_status_to_populated_raw_table = schedule( 'set_status_to_populated_raw_table', self.set_status_to_populated_raw_table, requires=[load_staging_raw_table]) load_subscription_snapshot_monthly_table = schedule( 'load_subscription_snapshot_monthly_table', self.load_subscription_snapshot_monthly_table, requires=[set_status_to_populated_raw_table]) next_step_requires = [load_subscription_snapshot_monthly_table] if bootstrap.result.get('bootstrap.source') in \ [config.source_weekly, config.source_monthly]: delete_data_for_old_source = schedule( 'delete_data_for_old_source', self.delete_data_for_old_source, requires=[load_subscription_snapshot_monthly_table]) next_step_requires = [delete_data_for_old_source] schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=next_step_requires ) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', source='source', vendor_name='vendor_name'))) @property def reporter_to_s3(self): """Load Reporter files to apple podcasts reports S3 archive bucket.""" return self.create( name='reporter_to_s3', schedule_to_start=48000, tasks=base.SyncRunner( apple_podcasts_reporter_tasks.extract_reporter_file_to_s3.fill( namespace='reporter_to_s3', reporter_account=StaticParam( config.account_id), report_type=StaticParam(config.report_type), report_role=StaticParam('sales'), date='bootstrap.date', date_type='bootstrap.source', secrets_path=StaticParam( config.secrets_path), destination_s3_path='bootstrap.s3_archive_path', feed_name='bootstrap.feed_name', vendor_id='bootstrap.vendor_id'))) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_DOWNLOADED)))) @property def set_status_to_populated_raw_table(self): """Set overall feed status to POPULATED_RAW_TABLE.""" return self.create( name='set_status_to_populated_raw_table', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_populated_raw_table', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_POPULATED_RAW_TABLE)))) @property def load_subscription_snapshot_monthly_table(self): """Load sales summary data.""" return self.create( name='load_subscription_snapshot_monthly_table', tasks=base.SyncRunner( tasks.load_subscription_snapshot_monthly_table.fill( namespace='load_subscription_snapshot_monthly_table', feed_name='bootstrap.feed_name', date='bootstrap.date', source='bootstrap.source', vendor_id='bootstrap.vendor_id', vendor_name='bootstrap.vendor_name', staging_raw_table=StaticParam( config.staging_raw_table)))) @property def delete_data_for_old_source(self): """Delete entries which are from old source.""" return self.create( name='delete_data_for_old_source', tasks=base.SyncRunner( tasks.delete_data_for_old_source.fill( namespace='delete_data_for_old_source', feed_name='bootstrap.feed_name', date='bootstrap.date', source='bootstrap.source', vendor_id='bootstrap.vendor_id'))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam(garcon_feed_status.STATUS_INGESTED)))) load_staging_raw_table = \ ApplePodcastsSnapshotMonthlyReportsSL.load_activity( feed_name=config.overall_feed_name, secrets_path=config.secrets_path, task_timeout=600, sql_loader=SQLLoader(__file__), requirements=dict( feed_name='bootstrap.feed_name', date='bootstrap.date', source='bootstrap.source', vendor_id='bootstrap.vendor_id', vendor_name='bootstrap.vendor_name', source_files_dict='bootstrap.source_files_dict', s3_dir_path='bootstrap.s3_archive_path', staging_raw_table_name=StaticParam( config.staging_raw_table) ))