"""Chartmetric Tracks Ingestion Workflow.""" from garcon_contrib.dynamo_feed_status import garcon_feed_status as feed_status from feed_ingestion.flows import base from feed_ingestion.flows.chartmetric_tracks import config from feed_ingestion.flows.chartmetric_tracks import tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='0.1') self.timeout = 72000 * 2 def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) create_tables = schedule( 'create_tables', self.create_tables, requires=[bootstrap]) ingest_raw_data = schedule( 'ingest_raw_data', self.ingest_raw_data, requires=[create_tables]) delete_defunct_participations = schedule( 'delete_defunct_participations', self.delete_defunct_participations, requires=[ingest_raw_data]) schedule( 'set_overall_status_ingested', self.set_overall_status, requires=[delete_defunct_participations], input={'status': feed_status.STATUS_INGESTED}) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', full_refresh='full_refresh', date_limit='context_date_limit', platform_names='platform_names'))) @property def create_tables(self): """Create tables in Snowflake.""" return self.create( name='create_tables', generators=[self.platform_names_generator], tasks=base.AsyncRunner( tasks.create_table.fill( namespace='create_table', feed_name='bootstrap.feed_name', date='bootstrap.date', date_limit='bootstrap.date_limit', full_refresh='bootstrap.full_refresh', platform_name='platform_name'), max_workers=4)) @property def ingest_raw_data(self): """Ingest raw data from the Snowflake tables into Neo4j.""" return self.create( name='ingest_raw_data', tasks=base.SyncRunner( tasks.ingest_data.fill( namespace='ingest_raw_data', feed_name='bootstrap.feed_name', platform_names='bootstrap.platform_names'))) @property def delete_defunct_participations(self): """Delete defunct connections.""" return self.create( name='delete_defunct_participations', tasks=base.SyncRunner( tasks.delete_defunct_participations.fill( feed_name='bootstrap.feed_name', date='bootstrap.date'))) def platform_names_generator(self, context): """Generate parameters for ingestion activities. Args: context (dict): The current context. Yields: dict: Dictionary with a platform name. """ for plaform_name in context['bootstrap.platform_names']: yield dict(platform_name=plaform_name)