"""YouTube Facts Workflow.""" import datetime from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.youtube_facts import config from feed_ingestion.flows.youtube_facts import tasks from feed_ingestion.tasks import load_fact_tables_tasks_sf, \ overall_status_tasks class Flow(base.FlowBase, base.FlowConfigMixin, base.FlowLoadFactMixinSF): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return check_staging_status = schedule( 'check_staging_status', self.check_staging_status, requires=[bootstrap]) if check_staging_status.result.get( 'check_staging_status.stop') is True: return load_demographics_table = schedule( 'load_demographics_table', self.load_demographics_table, requires=[check_staging_status]) load_staging_fact_table = schedule( 'load_staging_fact_table', self.load_staging_fact_table, requires=[load_demographics_table]) load_mapping_table = schedule( 'load_mapping_table', self.load_mapping_table, requires=[load_staging_fact_table]) schedule( 'load_fact_tables', self.load_fact_tables, requires=[load_mapping_table]) def workflow_id(self, initial_context): """Generate workflow id. Args: initial_context (dict): The initial context for the flow. Returns: str: A unique identifier for a workflow being executed. """ if 'report_type' not in initial_context: raise ValueError('There is no report_type in context') if 'licensor' not in initial_context: raise ValueError('There is no licensor in context') report_type = initial_context['report_type'] flow_name = '_'.join([self.name, report_type]) licensor = initial_context['licensor'] if 'context_date' not in initial_context: date = datetime.datetime.today().strftime('%Y-%m-%d') else: date = initial_context['context_date'] return '{flow_name}-{licensor}-{date}'.format( flow_name=flow_name, licensor=licensor, date=date) def contextified_feed_name(self, context): """Get feed_name in context. Args: context (dict): The context of the flow. Returns: str: Contextified feed name. """ if 'report_type' not in context: raise ValueError('There is no report_type in context') if 'licensor' not in context: raise ValueError('There is no licensor in context') return '_'.join([self.feed_name, context['licensor'], context['report_type']]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', report_type='report_type', licensor='licensor', reload='reload'))) @property def check_staging_status(self): """Check if all required reports is available.""" return self.create( name='check_staging_status', tasks=base.SyncRunner( tasks.check_staging_status.fill( namespace='check_staging_status', date='bootstrap.date', report_type='bootstrap.report_type', licensor='bootstrap.licensor'))) @property def load_staging_fact_table(self): """Activity to load staging_fact_analytics_{feed_name}_{datestamp}.""" return self.create( name='unload_fact_data', schedule_to_start=48000, tasks=base.SyncRunner( load_fact_tables_tasks_sf.create_staging_fact.fill( namespace='create_staging_fact', feed_name='bootstrap.report_status_name', secrets_path=StaticParam(config.secrets_path), date='bootstrap.date', sfdb_params='sfdb_params'), load_fact_tables_tasks_sf.load_staging_fact.fill( namespace='load_staging_fact', feed_name='bootstrap.report_status_name', secrets_path=StaticParam(config.secrets_path), date='bootstrap.date', sfdb_params='sfdb_params'))) @property def load_fact_tables(self): """Activity to load fact_analytics & fact_analytics_error. Deletes any existing data for this feed and date in fact_analytics & fact_analytics_error. After successfully loading the data, sets the feed's status for that download_date to INGESTED. """ activity_name = 'load_fact_tables' return self.create( name=activity_name, tasks=base.SyncRunner( load_fact_tables_tasks_sf.load_fact_data.fill( namespace='load_fact_data', feed_name='bootstrap.report_status_name', secrets_path=StaticParam(config.secrets_path), date='{}.date'.format(self.bootstrap_namespace), sfdb_params='sfdb_params', kwargs='bootstrap.kwargs'), overall_status_tasks.set_overall_status.fill( feed_name='bootstrap.report_status_name', date='{}.date'.format(self.bootstrap_namespace), set_status_once=StaticParam(True), status=StaticParam(garcon_feed_status.STATUS_INGESTED)))) @property def load_demographics_table(self): """Activity to load demographics table.""" activity_name = 'load_demographics_table' return self.create( name=activity_name, tasks=base.SyncRunner( tasks.load_demographics_table.fill( namespace='load_demographics_table', feed_name='bootstrap.report_status_name', date='{}.date'.format(self.bootstrap_namespace), sfdb_params='sfdb_params', kwargs='bootstrap.kwargs'))) @property def load_mapping_table(self): """Activity to load video_asset_mapping table.""" activity_name = 'load_mapping_table' return self.create( name=activity_name, tasks=base.SyncRunner( tasks.load_mapping_table.fill( namespace='load_mapping_table', feed_name='bootstrap.report_status_name', report_type='report_type', date='{}.date'.format(self.bootstrap_namespace), sfdb_params='sfdb_params', kwargs='bootstrap.kwargs')))