"""Apple Financial Workflow.""" from garcon import param from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.apple_financial import config from feed_ingestion.flows.apple_financial import generators from feed_ingestion.flows.apple_financial import tasks from feed_ingestion.flows.base import FlowBase from feed_ingestion.flows.base import FlowConfigMixin from feed_ingestion.flows.base import FlowLoadMarketshareMixinSF from feed_ingestion.tasks import date_tasks from feed_ingestion.tasks import feed_status_tasks from feed_ingestion.tasks import load_raw_table_tasks_sf from feed_ingestion.tasks import overall_status_tasks from feed_ingestion.tasks import reporter_tasks class Flow(FlowBase, FlowConfigMixin, FlowLoadMarketshareMixinSF): """Apple Financial Flow class. This flow downloads Apple Marketshare data and loads to snowflake. """ def __init__(self): """Initialize an Apple Financial workflow.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule, context): """Orchestrate an Apple Financial workflow. Args: schedule (callable): The scheduler method. context (dict): Initial context of workflow. """ get_first_day_of_month = schedule( 'get_first_day_of_month', self.get_first_day_of_month) bootstrap = schedule( 'bootstrap', self.bootstrap, requires=[get_first_day_of_month]) reporter_to_s3 = schedule( 'reporter_to_s3', self.reporter_to_s3, requires=[bootstrap]) check_files_on_s3 = schedule( 'check_files_on_s3', self.check_files_on_s3, requires=[reporter_to_s3]) if check_files_on_s3.result.get('check_files_on_s3.stop') is True: return reset_dynamo_db_status = schedule( 'reset_dynamo_db_status', self.reset_dynamo_db_status, requires=[check_files_on_s3]) process_drop_files = schedule( 'process_drop_files', self.process_drop_files, requires=[reset_dynamo_db_status]) create_temp_staging_raw_tables = schedule( 'create_temp_staging_raw_tables', self.create_temp_staging_raw_tables, requires=[process_drop_files]) load_temp_staging_raw_tables = schedule( 'load_temp_staging_raw_tables', self.load_temp_staging_raw_tables, requires=[create_temp_staging_raw_tables]) populate_staging_raw = schedule( 'populate_staging_raw', self.populate_staging_raw, requires=[load_temp_staging_raw_tables]) drop_temp_stage_tables = schedule( 'drop_temp_stage_tables', self.drop_temp_stage_tables, requires=[populate_staging_raw]) mark_ingested_files = schedule( 'mark_ingested_files', self.mark_ingested_files, requires=[drop_temp_stage_tables] ) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[mark_ingested_files], ) @property def get_first_day_of_month(self): """Get the first day of the month of date.""" return self.create( name='get_first_day_of_month', tasks=base.SyncRunner( date_tasks.get_first_day_of_month.fill( namespace='get_first_day_of_month', date='context_date'))) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='get_first_day_of_month.date', reload='reload'))) @property def check_files_on_s3(self): """Check if there are some new files in s3_download_path.""" return self.create( name='check_files_on_s3', tasks=base.SyncRunner( feed_status_tasks.check_files_on_s3.fill( namespace='check_files_on_s3', feed_name='bootstrap.feed_name', date='bootstrap.date', file_pattern='bootstrap.file_pattern', s3_download_path='bootstrap.s3_archive_path'))) @property def reporter_to_s3(self): """Load Reporter files to Apple Music Streams' S3 archives bucket.""" return self.create( name='reporter_to_s3', generators=[generators.itunes_reports_generator], schedule_to_start=14400, tasks=base.SyncRunner( reporter_tasks.extract_reporter_file_to_s3.fill( namespace='reporter_to_s3', reporter_account='generator.reporter_account', report_type='generator.report_type', report_country='generator.report_country', report_role=param.StaticParam('finance'), date='bootstrap.date', destination_s3_path='bootstrap.s3_archive_path', feed_name=param.StaticParam(self.feed_name)))) @property def mark_ingested_files(self): """Put list of ingested files in DynamoDB.""" return self.create( name='mark_ingested_files', tasks=base.SyncRunner( feed_status_tasks.mark_ingested_files.fill( namespace='mark_ingested_files', feed_name='bootstrap.feed_name', date='bootstrap.date', source_files_dict='check_files_on_s3.source_files_dict'))) @property def process_drop_files(self): """Process and archive drop files on S3.""" return self.create( name='process_drop_files', generators=[generators.process_drop_files_generator], tasks=base.SyncRunner( tasks.process_drop_files.fill( namespace='process_drop_files', feed_name='bootstrap.feed_name', date='bootstrap.date', s3_archive_path='bootstrap.s3_archive_path', files='check_files_on_s3.source_files_dict', ))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=base.SyncRunner( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name='bootstrap.feed_name', date='bootstrap.date', set_status_once=StaticParam(True), status=StaticParam(garcon_feed_status.STATUS_INGESTED), ), ), ) @property def create_temp_staging_raw_tables(self): """Create temp staging raw table.""" return self.create( name='create_temp_staging_raw_tables', generators=[generators.temp_staging_raw_generator], schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.create_temp_staging_raw_table.fill( namespace='create_temp_staging_raw_tables', date='bootstrap.date', sfdb_params='sfdb_params', feed_name='feed_name', secrets_path=param.StaticParam(config.secrets_path), temp_staging_raw_table='temp_table_name', kwargs='kwargs'))) @property def load_temp_staging_raw_tables(self): """Load temp staging raw tables.""" return self.create( name='load_temp_staging_raw_tables', generators=[generators.temp_staging_raw_generator], schedule_to_start=48000, tasks=base.SyncRunner( load_raw_table_tasks_sf.load_temp_staging_raw_table.fill( namespace='load_temp_staging_raw_tables', aws=param.StaticParam(self.conf_aws), date='bootstrap.date', sfdb_params='sfdb_params', feed_name='feed_name', secrets_path=param.StaticParam(config.secrets_path), kwargs='kwargs', key_dir='key_dir', temp_staging_raw_table='temp_table_name', error_limit='bootstrap.snowflake_error_limit'))) @property def populate_staging_raw(self): """Join temp tables and unload data in the staging_raw table.""" return self.create( name='load_staging_raw', generators=[generators.populate_staging_raw_generator], schedule_to_start=48000, tasks=base.SyncRunner( tasks.load_staging_raw_table.fill( namespace='load_staging_raw_table', date='bootstrap.date', processed_datetime='bootstrap.processed_datetime', vendor_id='vendor_id', filename='filename', sfdb_params='sfdb_params', staging_raw_table='staging_raw_table', report_name='report_name', feed_name='feed_name', secrets_path=param.StaticParam(config.secrets_path), licensor='licensor'))) @property def drop_temp_stage_tables(self): """Drop the staging_raw temp tables.""" return self.create( name='drop_temp_stage_tables', generators=[generators.temp_staging_raw_generator], schedule_to_start=48000, tasks=base.SyncRunner( tasks.drop_temp_table.fill( namespace='drop_temp_table', temp_table_name='temp_table_name', sfdb_params='sfdb_params', secrets_path=param.StaticParam(config.secrets_path), report_name='report_name')))