"""Chartmetric Socials Ingestion Workflow.""" from garcon.param import StaticParam from garcon_contrib.dynamo_feed_status import garcon_feed_status as feed_status from feed_ingestion.flows import base from feed_ingestion.flows.chartmetric_socials import config from feed_ingestion.flows.chartmetric_socials import tasks class Flow(base.FlowBase, base.FlowConfigMixin): """Class representing the workflow.""" def __init__(self): """Initialize flow object.""" super(Flow, self).__init__(feed_name=config.feed_name, version='0.1') self.timeout = 60*60*6 def decider(self, schedule, context): """Activity decider. Args: schedule (callable): The scheduler method. """ bootstrap = schedule('bootstrap', self.bootstrap) clear_log_table = schedule( 'clear_log_table', self.clear_log_table, requires=[bootstrap]) create_accounts_tables = schedule( 'create_accounts_tables', self.create_accounts_tables, requires=[clear_log_table]) ingest_socials_accounts = schedule( 'ingest_socials_accounts', self.ingest_socials_accounts, requires=[create_accounts_tables]) delete_created_pending_social_accounts = schedule( 'delete_created_pending_social_accounts', self.delete_created_pending_social_accounts, requires=[ingest_socials_accounts]) delete_old_pending_social_accounts = schedule( 'delete_old_pending_social_accounts', self.delete_old_pending_social_accounts, requires=[delete_created_pending_social_accounts]) create_temp_aggregates_tables = schedule( 'create_temp_aggregates_tables', self.create_temp_aggregates_tables, requires=[delete_old_pending_social_accounts]) create_aggregate_by_account_table = schedule( 'create_aggregate_by_account_table', self.create_aggregate_by_account_table, requires=[create_temp_aggregates_tables]) create_temp_aggregate_by_participant_table = schedule( 'create_temp_aggregate_by_participant_table', self.create_temp_aggregate_by_participant_table, requires=[create_temp_aggregates_tables]) populate_temp_aggregate_by_participant_table = schedule( 'populate_temp_aggregate_by_participant_table', self.populate_temp_aggregate_by_participant_table, requires=[create_temp_aggregate_by_participant_table]) create_main_aggregate_by_participant_table = schedule( 'create_main_aggregate_by_participant_table', self.create_main_aggregate_by_participant_table, requires=[populate_temp_aggregate_by_participant_table]) populate_aggregate_by_account_table = schedule( 'populate_aggregate_by_account_table', self.populate_aggregate_by_account_table, requires=[create_aggregate_by_account_table]) ingest_aggregate_socials_by_account = schedule( 'ingest_aggregate_socials_by_account', self.ingest_aggregate_socials_by_account, requires=[populate_aggregate_by_account_table]) ingest_aggregate_socials_by_participant = schedule( 'ingest_aggregate_socials_by_participant', self.ingest_aggregate_socials_by_participant, requires=[ ingest_aggregate_socials_by_account, create_main_aggregate_by_participant_table]) insert_fact_socials = schedule( 'insert_fact_socials', self.insert_fact_socials, requires=[bootstrap]) insert_into_log_table = schedule( 'insert_into_log_table', self.insert_into_log_table, requires=[ insert_fact_socials, ingest_aggregate_socials_by_participant, ingest_aggregate_socials_by_account]) create_fact_social_latest = schedule( 'create_fact_social_latest', self.create_fact_social_latest, requires=[insert_into_log_table]) schedule( 'set_overall_status_ingested', self.set_overall_status, requires=[create_fact_social_latest], input={'status': feed_status.STATUS_INGESTED}) @property def bootstrap(self): """Bootstrap initial configuration.""" return self.create( name='bootstrap', tasks=base.SyncRunner( tasks.bootstrap.fill( namespace='bootstrap', date='context_date', date_limit='context_date_limit', platform_names='platform_names', reload='reload'))) @property def clear_log_table(self): """Clear log table in case of force refresh.""" return self.create( name='clear_log_table', generators=[self.log_table_generator], tasks=base.AsyncRunner( tasks.clear_log_table.fill( namespace='clear_log_table', feed_name='bootstrap.feed_name', platform_name='platform_name', reload='bootstrap.reload'), max_workers=4)) @property def create_accounts_tables(self): """Create accounts tables in Snowflake.""" return self.create( name='create_accounts_table', generators=[self.platform_names_generator], tasks=base.AsyncRunner( tasks.create_accounts_table.fill( namespace='create_accounts_table', feed_name='bootstrap.feed_name', platform_name='platform_name'), max_workers=4)) @property def ingest_socials_accounts(self): """Ingest Socials accounts.""" return self.create( name='ingest_socials_accounts', generators=[self.platform_names_generator], tasks=base.SyncRunner( tasks.ingest_social_accounts.fill( namespace='ingest_socials_accounts', feed_name='bootstrap.feed_name', platform_name='platform_name' ) ) ) @property def delete_created_pending_social_accounts(self): """Remove Pending Social Accounts that have been created.""" return self.create( name='delete_created_pending_social_accounts', tasks=base.SyncRunner( tasks.delete_pending_social_accounts.fill( namespace='delete_pending_social_accounts', feed_name='bootstrap.feed_name', query_name=StaticParam( 'delete_created_pending_social_accounts' ) ) ) ) @property def delete_old_pending_social_accounts(self): """Remove Pending Social Accounts older than 96 days.""" return self.create( name='delete_old_pending_social_accounts', tasks=base.SyncRunner( tasks.delete_pending_social_accounts.fill( namespace='delete_pending_social_accounts', feed_name='bootstrap.feed_name', query_name=StaticParam( 'delete_old_pending_social_accounts' ) ) ) ) @property def create_temp_aggregates_tables(self): """Create temp aggregates tables.""" return self.create( name='create_temp_aggregates_tables', generators=[self.aggregate_platform_names_generator], tasks=base.AsyncRunner( tasks.create_temp_aggregates_table.fill( namespace='create_temp_aggregates_table', platform_name='platform_name' ), max_workers=4 ) ) @property def create_aggregate_by_account_table(self): """Create aggregate_by_account table in Snowflake.""" return self.create( name='create_aggregate_by_account_table', tasks=base.SyncRunner( tasks.create_aggregate_by_account_table.fill( namespace='create_aggregate_by_account_table'))) @property def create_temp_aggregate_by_participant_table(self): """Create temp aggregate_by_participant tables in Snowflake.""" return self.create( name='create_temp_aggregate_by_participant_table', tasks=base.SyncRunner( tasks.create_temp_aggregate_by_participant_table.fill( namespace='create_temp_aggregate_by_participant_table') ) ) @property def populate_temp_aggregate_by_participant_table(self): """Populate temp aggregate_by_participant table.""" return self.create( name='populate_temp_aggregate_by_participant_table', generators=[self.aggregate_platform_names_generator], tasks=base.AsyncRunner( tasks.populate_temp_aggregate_by_participant_table.fill( namespace='populate_temp_aggregate_by_participant_table', platform_name='platform_name' ), max_workers=4 ) ) @property def create_main_aggregate_by_participant_table(self): """Create aggregate_by_participant tables in Snowflake.""" return self.create( name='create_main_aggregate_by_participant_table', tasks=base.SyncRunner( tasks.create_main_aggregate_by_participant_table.fill( namespace='create_main_aggregate_by_participant_table') ) ) @property def populate_aggregate_by_account_table(self): """Populate main aggregate_by_account table.""" return self.create( name='populate_aggregate_by_account_table', generators=[self.aggregate_platform_names_generator], tasks=base.AsyncRunner( tasks.populate_aggregate_by_account_table.fill( namespace='populate_aggregate_by_account_table', platform_name='platform_name' ), max_workers=4 ) ) @property def ingest_aggregate_socials_by_account(self): """Ingest aggregated data by account.""" return self.create( name='ingest_aggregate_socials', tasks=base.SyncRunner( tasks.ingest_aggregate_social_data.fill( namespace='ingest_aggregate_socials', feed_name='bootstrap.feed_name', query_name=StaticParam( 'ingest_aggregate_socials_by_account') ) ) ) @property def ingest_aggregate_socials_by_participant(self): """Ingest aggregated data by participant.""" return self.create( name='ingest_aggregate_socials', tasks=base.SyncRunner( tasks.ingest_aggregate_social_data.fill( namespace='ingest_aggregate_socials', feed_name='bootstrap.feed_name', query_name=StaticParam( 'ingest_aggregate_socials_by_participant') ) ) ) @property def insert_into_log_table(self): """Insert unique keys into the log table in Snowflake.""" return self.create( name='insert_into_log_table', generators=[self.log_table_generator], tasks=base.AsyncRunner( tasks.insert_into_log_table.fill( namespace='insert_into_log_table', feed_name='bootstrap.feed_name', ingestion_started_at='bootstrap.ingestion_started_at', platform_name='platform_name'), max_workers=4)) @property def create_fact_social_latest(self): """Insert unique keys into the log table in Snowflake.""" return self.create( name='create_fact_social_latest', tasks=base.SyncRunner( tasks.create_fact_social_latest.fill( namespace='create_fact_social_latest', ) ) ) @property def insert_fact_socials(self): """Insert data into fact_socials table in snowflake.""" return self.create( name='insert_fact_socials', generators=[self.platform_names_generator], tasks=base.SyncRunner( tasks.insert_fact_socials.fill( namespace='insert_fact_socials', feed_name='bootstrap.feed_name', date='bootstrap.date', date_limit='bootstrap.date_limit', ingestion_started_at='bootstrap.ingestion_started_at', reload_artist_with_no_stat='reload_artist_with_no_stat', platform_name='platform_name'))) def platform_names_generator(self, context): """Generate parameters for ingestion activities. Args: context (dict): The current context. Yields: dict: Dictionary with a platform name. """ for platform_name in context['bootstrap.platform_names']: yield dict(platform_name=platform_name) def aggregate_platform_names_generator(self, context): """Generate parameters for aggregation related activities. Yields: dict: Dictionary with a platform name. """ # TODO: it should respect platfrom_names from bootstrap context for platform_name in config.all_platform_names: yield dict(platform_name=platform_name) def log_table_generator(self, context): """Generate parameters for clear_log_table. Args: context (dict): The current context. Yields: dict: Dictionary with a platform name. """ all_platform_names = context[ 'bootstrap.platform_names'] + [ 'aggregate_by_participant', 'aggregate_by_account'] for platform_name in all_platform_names: yield dict(platform_name=platform_name)