""" Apple podcast sales summary weekly data ingestion workflow. Extract, transform and load Apple podcast subscription sales summary weekly report raw data to apple_podcasts_sales_summary_weekly table. """ from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows import base from feed_ingestion.flows.apple_podcasts_sales_summary_weekly import config from feed_ingestion.flows.apple_podcasts_sales_summary_weekly import tasks from feed_ingestion.flows.apple_podcasts_sales_summary_weekly.stage_loader \ import ApplePodcastsWeeklyReportsSL from feed_ingestion.flows.base import FlowBase from feed_ingestion.flows.base import FlowConfigMixin from feed_ingestion.tasks import apple_podcasts_reporter_tasks from feed_ingestion.tasks import overall_status_tasks class Flow(FlowBase, FlowConfigMixin): """Class representing a workflow.""" def __init__(self): """Initialize flow object.""" super().__init__(feed_name=config.overall_feed_name, version='2.0') self.timeout = 21600 # override default to 6 hours (60 *60 * 6) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop'): return reporter_to_s3 = schedule( 'reporter_to_s3', self.reporter_to_s3, requires=[bootstrap]) if reporter_to_s3.result.get('reporter_to_s3.file_name')\ != bootstrap.result.get('bootstrap.expected_file_name') or \ (reporter_to_s3.result.get('reporter_to_s3.file_name') == bootstrap.result.get('bootstrap.expected_file_name') and reporter_to_s3.result.get('reporter_to_s3.status') != garcon_feed_status.STATUS_DOWNLOADED): return set_status_to_downloaded = schedule( 'set_status_to_downloaded', self.set_status_to_downloaded, requires=[reporter_to_s3]) load_staging_raw_table = schedule( 'load_staging_raw_table', self.load_staging_raw_table, requires=[set_status_to_downloaded]) set_status_to_populated_raw_table = schedule( 'set_status_to_populated_raw_table', self.set_status_to_populated_raw_table, requires=[load_staging_raw_table]) apple_podcasts_sales_summary_weekly = schedule( 'load_sales_summary_weekly_table', self.load_sales_summary_weekly_table, requires=[set_status_to_populated_raw_table]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[apple_podcasts_sales_summary_weekly] ) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload', vendor_name='vendor_name'))) @property def reporter_to_s3(self): """Load Reporter files to apple podcasts reports S3 archive bucket.""" return self.create( name='reporter_to_s3', schedule_to_start=48000, tasks=base.SyncRunner( apple_podcasts_reporter_tasks.extract_reporter_file_to_s3.fill( namespace='reporter_to_s3', reporter_account=StaticParam( config.account_id), report_type=StaticParam(config.report_type), report_role=StaticParam('sales'), date='bootstrap.date', secrets_path=StaticParam( config.secrets_path), destination_s3_path='bootstrap.s3_archive_path', feed_name='bootstrap.feed_name', vendor_id='bootstrap.vendor_id'))) @property def set_status_to_downloaded(self): """Set overall feed status to DOWNLOADED.""" return self.create( name='set_status_to_downloaded', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_downloaded', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_DOWNLOADED)))) @property def set_status_to_populated_raw_table(self): """Set overall feed status to POPULATED_RAW_TABLE.""" return self.create( name='set_status_to_populated_raw_table', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_populated_raw_table', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam( garcon_feed_status.STATUS_POPULATED_RAW_TABLE)))) @property def load_sales_summary_weekly_table(self): """Load sales summary weekly data.""" return self.create( name='load_sales_summary_weekly_table', tasks=base.SyncRunner( tasks.load_sales_summary_weekly_table.fill( namespace='load_sales_summary_weekly_table', feed_name='bootstrap.feed_name', date='bootstrap.date', vendor_id='bootstrap.vendor_id', vendor_name='bootstrap.vendor_name', staging_raw_table=StaticParam( config.staging_raw_table)))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', status=StaticParam(garcon_feed_status.STATUS_INGESTED)))) load_staging_raw_table = ApplePodcastsWeeklyReportsSL.load_activity( feed_name=config.overall_feed_name, secrets_path=config.secrets_path, task_timeout=600, sql_loader=SQLLoader(__file__), requirements=dict( date='bootstrap.date', vendor_id='bootstrap.vendor_id', vendor_name='bootstrap.vendor_name', source_files_dict='bootstrap.source_files_dict', s3_dir_path='bootstrap.s3_archive_path', staging_raw_table_name=StaticParam( config.staging_raw_table) ))