"""YouTube Neo4j Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.youtube_neo4j import config from feed_ingestion.flows.youtube_neo4j import tasks from feed_ingestion.tasks import overall_status_tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(config.feed_name, config.feed_version) def decider(self, schedule): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) if bootstrap.result.get('bootstrap.stop') is True: return check_staging_status = schedule( 'check_staging_status', self.check_staging_status, requires=[bootstrap]) if check_staging_status.result.get( 'check_staging_status.stop') is True: return clear_log_table = schedule( 'clear_log_table', self.clear_log_table, requires=[check_staging_status]) create_tables = schedule( 'create_tables', self.create_tables, requires=[clear_log_table]) ingest_youtube_data = schedule( 'ingest_youtube_data', self.ingest_youtube_data, requires=[create_tables]) schedule( 'insert_into_log_table', self.insert_into_log_table, requires=[ingest_youtube_data]) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload'))) @property def check_staging_status(self): """Check if all required reports is available.""" return self.create( name='check_staging_status', tasks=base.SyncRunner( tasks.check_staging_status.fill( namespace='check_staging_status', date='bootstrap.date'))) @property def clear_log_table(self): """Clear log table in case of force refresh.""" return self.create( name='clear_log_table', tasks=base.AsyncRunner( tasks.clear_log_table.fill( namespace='clear_log_table', feed_name='bootstrap.feed_name', date='bootstrap.date', reload='reload'), max_workers=4)) @property def create_tables(self): """Create tables in Snowflake.""" return self.create( name='create_tables', generators=[self.query_names_generator], tasks=base.SyncRunner( tasks.create_table.fill( namespace='create_table', feed_name='bootstrap.feed_name', date='bootstrap.date', query_name='query_name', temp_table='temp_table', full_owner_changed_refresh='full_owner_changed_refresh'))) @property def ingest_youtube_data(self): """Ingest YouTube data into Neo4j.""" return self.create( name='ingest_youtube_data', tasks=base.SyncRunner( tasks.ingest_youtube_data.fill( namespace='ingest_youtube_data', feed_name='bootstrap.feed_name', date='bootstrap.date', query_type=StaticParam('channel'), temp_table=StaticParam( f'temp_{config.feed_name}_channel')), tasks.ingest_youtube_data.fill( namespace='ingest_youtube_data', feed_name='bootstrap.feed_name', date='bootstrap.date', query_type=StaticParam('video'), temp_table=StaticParam( f'temp_{config.feed_name}_video')), tasks.ingest_youtube_data.fill( namespace='ingest_youtube_data', feed_name='bootstrap.feed_name', date='bootstrap.date', query_type=StaticParam('asset'), temp_table=StaticParam( f'temp_{config.feed_name}_asset')))) @property def insert_into_log_table(self): """Insert unique keys into the log table in Snowflake.""" return self.create( name='insert_into_log_table', tasks=base.SyncRunner( tasks.insert_into_log_table.fill( namespace='insert_into_log_table', feed_name='bootstrap.feed_name', date='bootstrap.date', query_name=StaticParam('video'), temp_table=StaticParam( f'temp_{config.feed_name}_video')), tasks.insert_into_log_table.fill( namespace='insert_into_log_table', feed_name='bootstrap.feed_name', date='bootstrap.date', query_name=StaticParam('asset'), temp_table=StaticParam( f'temp_{config.feed_name}_asset')), overall_status_tasks.set_overall_status.fill( feed_name='bootstrap.feed_name', date='bootstrap.date', set_status_once=StaticParam(True), status=StaticParam(garcon_feed_status.STATUS_INGESTED)))) def query_names_generator(self, context): """Generate parameters for ingestion activities. Args: context (dict): The current context. Yields: dict: Dictionary with a platform name. """ for query_name in config.query_names: yield { 'query_name': query_name, 'temp_table': f'temp_{context["bootstrap.feed_name"]}_{query_name}' }