"""Chartmetric Socials Ingestion Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.flows import base from feed_ingestion.flows.chartmetric_socials_backfill import config from feed_ingestion.flows.chartmetric_socials_backfill import tasks from feed_ingestion.tasks import overall_status_tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='0.1') self.timeout = 16200 * 5 def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) delete_removed_accounts = schedule( 'delete_removed_accounts', self.delete_removed_accounts, requires=[bootstrap]) get_modified_accounts = schedule( 'get_modified_accounts', self.get_modified_accounts, requires=[delete_removed_accounts]) create_table = schedule( 'create_table', self.create_table, requires=[get_modified_accounts]) populate_table = schedule( 'populate_table', self.populate_table, requires=[create_table]) ingest_aggregate_socials = schedule( 'ingest_aggregate_socials', self.ingest_aggregate_socials, requires=[populate_table]) update_fact_socials = schedule( 'update_fact_socials', self.update_fact_socials, requires=[ingest_aggregate_socials]) save_new_accounts = schedule( 'save_new_accounts', self.save_new_accounts, requires=[update_fact_socials]) drop_temp_table = schedule( 'drop_temp_table', self.drop_temp_table, requires=[save_new_accounts]) schedule( 'set_overall_status_ingested', self.set_overall_status, requires=[drop_temp_table], input={'status': garcon_feed_status.STATUS_INGESTED}) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', date_from='context_date_from', platform_names='platform_names', reload='reload'))) @property def delete_removed_accounts(self): """Delete accounts from Neo4j that were deleted in Chartmetric.""" return self.create( name='delete_removed_accounts', generators=[self.platform_names_generator], tasks=base.SyncRunner( tasks.delete_removed_accounts.fill( namespace='delete_removed_accounts', feed_name='bootstrap.feed_name', platform_name='platform_name' ) ) ) @property def get_modified_accounts(self): """Get modified Socials accounts.""" return self.create( name='get_modified_accounts', tasks=base.SyncRunner( tasks.get_modified_accounts.fill( namespace='get_modified_accounts', feed_name='bootstrap.feed_name', date='bootstrap.date', date_from='bootstrap.date_from', reload='bootstrap.reload'))) @property def create_table(self): """Create snowflake aggregation table.""" return self.create( name='create_table', tasks=base.SyncRunner( tasks.create_table.fill( namespace='create_table', feed_name='bootstrap.feed_name'))) @property def populate_table(self): """Populate snowflake aggregation table.""" return self.create( name='populate_table', generators=[self.platform_names_generator], tasks=base.AsyncRunner( tasks.populate_table.fill( namespace='populate_table', feed_name='bootstrap.feed_name', target_ids_table='get_modified_accounts.target_ids_table', platform_name='platform_name' ), max_workers=4 ) ) @property def ingest_aggregate_socials(self): """Ingest aggregated data from snowflake.""" return self.create( name='ingest_aggregate_socials', tasks=base.SyncRunner( tasks.ingest_aggregate_social_data.fill( namespace='ingest_aggregate_socials', feed_name='bootstrap.feed_name', date='bootstrap.date', query_name=StaticParam( 'ingest_aggregate_socials_by_participant') ), tasks.ingest_aggregate_social_data.fill( namespace='ingest_aggregate_socials', feed_name='bootstrap.feed_name', date='bootstrap.date', query_name=StaticParam( 'ingest_aggregate_socials_by_account') ), overall_status_tasks.set_overall_status.fill( feed_name='bootstrap.feed_name', date='bootstrap.date', set_status_once=StaticParam(True), status=StaticParam(garcon_feed_status.STATUS_INGESTED) ) ) ) @property def save_new_accounts(self): """Insert modified accounts into log table.""" return self.create( name='save_new_accounts', tasks=base.SyncRunner( tasks.save_new_accounts.fill( namespace='save_new_accounts', target_ids_table='get_modified_accounts.target_ids_table', ingestion_started_at='bootstrap.ingestion_started_at' ) ) ) @property def update_fact_socials(self): """Update fact_socials table with correct urls.""" return self.create( name='update_fact_socials', generators=[self.platform_names_generator], tasks=base.SyncRunner( tasks.update_fact_socials.fill( namespace='update_fact_socials', feed_name='bootstrap.feed_name', date='bootstrap.date', target_ids_table='get_modified_accounts.target_ids_table', platform_name='platform_name' ) ) ) @property def drop_temp_table(self): """Drop the temp target_ids_table.""" return self.create( name='drop_temp_table', schedule_to_start=48000, tasks=base.SyncRunner( tasks.drop_temp_table.fill( namespace='drop_temp_table', temp_table_name='get_modified_accounts.target_ids_table'))) def platform_names_generator(self, context): """Generate parameters for ingestion activities. Args: context (dict): The current context. Yields: dict: Dictionary with a platform name. """ for plaform_name in context['bootstrap.platform_names']: yield dict(platform_name=plaform_name)