""" Source of Streams pre-aggregation workflow for Amazon Unlimited data. The workflow aggregates and loads Amazon Unlimited data to the staging_sos table. All data manipulations are performed within Snowflake warehouse. """ from garcon import runner from garcon.contrib.dynamo_feed_status import \ feed_status_ingestion as feed_status from garcon.param import StaticParam from analytics_aggregation.flows import base from analytics_aggregation.flows.amazon_unlimited_sos import config from analytics_aggregation.flows.amazon_unlimited_sos import tasks from analytics_aggregation.tasks import common as common_tasks from analytics_aggregation.tasks import overall_status_tasks class Flow(base.FlowBase): """Class representing a workflow.""" timeout = 4 * 60 * 60 # 4 hours def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.FEED_NAME, version='1.0') def decider(self, schedule, context=None): """Activity decider. Arg: schedule (callable): The scheduler method. context (dict): Initial context of workflow. """ bootstrap = schedule( 'bootstrap', self.bootstrap) # Stop flow if feed already ingested if bootstrap.result.get('bootstrap.stop'): return check_source_amazon_unlimited_data_ingested = schedule( 'check_sos_reports_ingested', self.check_source_amazon_unlimited_data_ingested, requires=[bootstrap]) # Stop flow when raw data is not ready if check_source_amazon_unlimited_data_ingested.result.get( 'check_source_amazon_unlimited_data_ingested.stop'): return populate_staging_sos = schedule( 'populate_staging_sos', self.populate_staging_sos, requires=[check_source_amazon_unlimited_data_ingested]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[populate_staging_sos]) def workflow_id(self, initial_context): """Generate workflow id. To avoid two flow execute simultaneously we should create all flows with the same workflow_id. Args: initial_context (dict): The initial context for the flow. Returns: str: Flow identifier in the forms of ''. """ return self.name @property def bootstrap(self): """Bootstrap the configuration.""" return self.create( name='bootstrap', tasks=runner.Sync( tasks.bootstrap.fill( namespace='bootstrap', feed_name=StaticParam(self.feed_name), source_feed_name=StaticParam(config.SOURCE_FEED_NAME), context_date_range='context_date_range', days_back=StaticParam(config.DAYS_BACK_FOR_INGESTION), reload='reload', labelids='labelids'))) @property def check_source_amazon_unlimited_data_ingested(self): """Check if source amazon data was ingested. This is useful for reload runs only (when context date was explicitly passed). """ return self.create( name='check_source_amazon_unlimited_data_ingested', tasks=runner.Sync( common_tasks.check_feed_ingested.fill( namespace='check_source_amazon_unlimited_data_ingested', source_feed_name=StaticParam(config.SOURCE_FEED_NAME), date_range='bootstrap.date_range'))) @property def populate_staging_sos(self): """Populate staging_sos table with AMS data. (For a flow run day, or date range in case of backfilling). """ return self.create( name='populate_staging_sos', tasks=runner.Sync( tasks.cleanup_staging_sos.fill( namespace='populate_staging_sos', date_range='bootstrap.date_range', labelids='bootstrap.labelids'), tasks.populate_staging_sos.fill( namespace='populate_staging_sos', date_range='bootstrap.date_range', labelids='bootstrap.labelids'))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=runner.Sync( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name=StaticParam(self.feed_name), date_range='bootstrap.date_range_as_str', status=StaticParam(feed_status.STATUS_INGESTED))))