"""Chartmetric Participants Ingestion Workflow.""" from garcon_contrib.dynamo_feed_status import garcon_feed_status as feed_status from feed_ingestion.flows import base from feed_ingestion.flows.chartmetric_participants import config from feed_ingestion.flows.chartmetric_participants import tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='0.1') def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) clear_log_table = schedule( 'clear_log_table', self.clear_log_table, requires=[bootstrap]) create_temp_participants_table = schedule( 'create_temp_participants_table', self.create_temp_participants_table, requires=[clear_log_table]) create_main_participants_table = schedule( 'create_main_participants_table', self.create_main_participants_table, requires=[create_temp_participants_table]) ingest_participants = schedule( 'ingest_participants', self.ingest_participants, requires=[create_main_participants_table]) delete_defunct_relationships = schedule( 'delete_defunct_relationships', self.delete_defunct_relationships, requires=[ingest_participants]) insert_into_log_table = schedule( 'insert_into_log_table', self.insert_into_log_table, requires=[delete_defunct_relationships]) schedule( 'set_overall_status_ingested', self.set_overall_status, requires=[insert_into_log_table], input={'status': feed_status.STATUS_INGESTED}) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', reload='reload'))) @property def clear_log_table(self): """Clear log table if reload flag passed.""" return self.create( name='clear_log_table', tasks=base.SyncRunner( tasks.clear_log_table.fill( namespace='clear_log_table', reload='bootstrap.reload'))) @property def create_temp_participants_table(self): """Create temp_chartmetric_participants table.""" return self.create( name='create_temp_participants_table', tasks=base.SyncRunner( tasks.create_temp_participants_table.fill( namespace='create_temp_participants_table'))) @property def create_main_participants_table(self): """Create chartmetric_participants table.""" return self.create( name='create_main_participants_table', tasks=base.SyncRunner( tasks.create_main_participants_table.fill( namespace='create_main_participants_table'))) @property def ingest_participants(self): """Ingest participants from Snowflake into Neo4j.""" return self.create( name='ingest_participants', tasks=base.SyncRunner( tasks.ingest_participants.fill( namespace='ingest_participants', feed_name='bootstrap.feed_name' ) ) ) @property def delete_defunct_relationships(self): """Delete defunct relationships in Neo4j.""" return self.create( name='delete_defunct_relationships', tasks=base.SyncRunner( tasks.delete_defunct_relationships.fill( namespace='delete_defunct_relationships', feed_name='bootstrap.feed_name'))) @property def insert_into_log_table(self): """Insert unique keys into the log table in Snowflake.""" return self.create( name='insert_into_log_table', tasks=base.SyncRunner( tasks.insert_into_log_table.fill( namespace='insert_into_log_table', ingestion_started_at='bootstrap.ingestion_started_at')))