"""Source of Stream Spotify workflow.""" from garcon import runner from garcon.contrib.dynamo_feed_status import \ feed_status_ingestion as feed_status from garcon.param import StaticParam from analytics_aggregation.flows import base from analytics_aggregation.flows.spotify_sos import config from analytics_aggregation.flows.spotify_sos import tasks from analytics_aggregation.tasks import overall_status_tasks from analytics_aggregation.util import common as common_utils class Flow(base.FlowBase): """Class representing a workflow.""" timeout = 20 * 60 * 60 def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.FEED_NAME, version='2.0') def decider(self, schedule, context=None): """Activity decider. Arg: schedule (callable): The scheduler method. context (dict): Initial context of workflow. """ context = context or {} if 'reload' in context: context['reload'] = common_utils.get_bool_from_flag( context['reload']) bootstrap = schedule( 'bootstrap', self.bootstrap) # Stop flow if feed already ingested if bootstrap.result.get('bootstrap.stop'): return check_sos_reports_ingested = schedule( 'check_sos_reports_ingested', self.check_sos_reports_ingested, requires=[bootstrap]) # Stop flow when row data is not ready if check_sos_reports_ingested.result.get( 'check_spotify_ingested.stop'): return populate_staging_sos = schedule( 'populate_staging_sos', self.populate_staging_sos, requires=[check_sos_reports_ingested]) schedule( 'set_status_to_ingested', self.set_status_to_ingested, requires=[populate_staging_sos]) def workflow_id(self, initial_context): """Generate workflow id. To avoid two flow execute simultaneously we should create all flows with the same workflow_id. Args: initial_context (dict): The initial context for the flow. Returns: str: Flow identifier in the forms of ''. """ return self.name @property def bootstrap(self): """Bootstrap the configuration.""" return self.create( name='bootstrap', tasks=runner.Sync( tasks.extract_date.fill( namespace='bootstrap', feed_name=StaticParam(self.feed_name), context_date_range='context_date_range', reload='reload', labelids='labelids'), overall_status_tasks.check_workflow_status.fill( namespace='bootstrap', feed_name=StaticParam(self.feed_name), date_range_as_str='bootstrap.date_range_as_str', reload='bootstrap.reload'))) @property def check_sos_reports_ingested(self): """Check if source feeds are already ingested. This is useful for reload runs only (when context date was explicitly passed). """ return self.create( name='check_sos_reports_ingested', tasks=runner.Sync( tasks.check_spotify_ingested.fill( namespace='check_spotify_ingested', date_range='bootstrap.date_range', reload='bootstrap.reload'))) @property def populate_staging_sos(self): """Populate staging_sos table with Spotify and AMS data. (For a flow run day, or date range in case of backfilling). """ return self.create( name='populate_staging_sos', tasks=runner.Sync( tasks.cleanup_staging_sos.fill( namespace='populate_staging_with_spotify_data', date_range='bootstrap.date_range', labelids='bootstrap.labelids'), tasks.populate_staging_with_spotify_data.fill( namespace='populate_staging_with_spotify_data', date_range='bootstrap.date_range', labelids='bootstrap.labelids'), tasks.sanity_check_populate_staging_sos.fill( namespace='populate_staging_sos', reload='bootstrap.reload', date_range='bootstrap.date_range'))) @property def set_status_to_ingested(self): """Set overall feed status to INGESTED.""" return self.create( name='set_status_to_ingested', tasks=runner.Sync( overall_status_tasks.set_overall_status.fill( namespace='set_status_to_ingested', feed_name=StaticParam(self.feed_name), date_range='bootstrap.date_range_as_str', status=StaticParam(feed_status.STATUS_INGESTED))))