"""The Snowflake connector class for the common fact analytics tasks.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.base_executor import SnowflakeAWSExecutor # Load SQL templates sql_loader = SQLLoader(__file__) class SnowflakeSQLExecutorFA(SnowflakeAWSExecutor): """Helper class to abstract loading of fact tables. 'FA' in 'SnowflakeSQLExecutorFA' stands for 'fact analytics'. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate fact analytics loading methods. """ @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed, and a key of executors dict in feed_ingestion/common/__init__.py file. Returns: str: Feed name, e.g. 'deezer_daily_sf'. """ raise NotImplementedError() @property def feedid(self): """Id of the feed in the dim_feed table. Returns: int: feedid. """ raise NotImplementedError() @property def storeid(self): """Storeid of feed data. Should match dim_store and feed config value. Returns: integer: Feed's storeid. """ raise NotImplementedError() @property def staging_raw_table(self): """Name of the staging_raw table for the feed. Returns: str: staging_raw_{feed} table name. """ raise NotImplementedError() @property def fact_table(self): """Name of the fact analytics table for feed. 'fact_analytics' for all the vetted feeds. Optional name may be used to make the old and new versions of the flow work side by side within one schema or db, or for testing purposes. Returns: str: Fact analytics table name. """ return 'fact_analytics' @property def fact_error_table(self): """Name of the fact analytics error table for feed. 'fact_analytics_error' for all the vetted feeds. Optional name may be used to make the old and new versions of the flow work side by side within one schema or db, or for testing purposes. Returns: str: Fact analytics error table name. """ return 'fact_analytics_error' def staging_fact_table(self, date): """Get name of temp staging_fact_analytics_ table. Args: date (str): Date of the data being process (YYYY-MM-DD). Returns: str: Name of the staging_fact_analytics table for a feed. """ return 'staging_fact_analytics_{feed_name}_{date}'.format( feed_name=self.feed_name, date=date.replace('-', '')) def create_staging_fact_table(self, date): """Create temp staging fact_analytics table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ self.create_table_like( self.staging_fact_table(date), source_table=self.fact_table, source_db=self.sf_config['db'], source_schema=self.sf_config['schema'], transient=True) def load_staging_fact_table(self, date): """Load staging fact_analytics table from staging_raw table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ raise NotImplementedError() def _delete_from_fact_table(self, table, date, **kwargs): """Delete rows in fact table with the current run date. This is required for the workflow to be idempotent, and to avoid row duplication. Before we'll load rows for a specific day to the fact tables, we have to delete rows which were added by previous (allegedly unsuccessful workflow run). Args: table (str): Either fact_analytics, either fact_analytics_error. date (str): Date of the data being process (YYYY-MM-DD). kwargs (dict): Custom arguments. """ sql_template = sql_loader.load_query('delete_from_fact_table') params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_table=table, reportdate=date, feedid=self.feedid) params.update(kwargs) licensor = kwargs.get('licensor', False) storeid = kwargs.get('storeid', False) if licensor and storeid: sql_template = sql_loader.load_query( 'delete_from_fact_table_licensor') sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) return self.fetchone(sql_template, params=non_identifier_params)[0] def delete_from_fact_table(self, date, **kwargs): """Delete rows in fact_analytics table with the current run date. Args: date (str): Date of the data being process (YYYY-MM-DD). kwargs (dict): Custom arguments. """ self._delete_from_fact_table(self.fact_table, date, **kwargs) def delete_from_fact_error_table(self, date, **kwargs): """Delete rows in fact_analytics_error table. (With the current run date). Args: date (str): Date of the data being process (YYYY-MM-DD). kwargs (dict): Custom arguments. """ self._delete_from_fact_table(self.fact_error_table, date, **kwargs) def load_fact_data(self, date, **kwargs): """Load matched data into fact_analytics. This method is pretty generic, so it's implemented in base FA executor class. Args: date (str): Date of the data being process (YYYY-MM-DD). """ sql_template = sql_loader.load_query('load_fact_analytics') params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_table=self.fact_table, staging_fact_analytics_table=self.staging_fact_table(date), staging_raw_table=self.staging_raw_table, reportdate=date) sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) self.execute(sql_template, params=non_identifier_params) def load_fact_error_data(self, date): """Load unmatched data into fact_analytics_error. Args: date (str): Date of the data being process (YYYY-MM-DD). """ raise NotImplementedError() def _update_dimension_table(self, date, table_name, sql_loader, **kwargs): """Update dimension table with the new data. Args: date (str): Date of the data being process (YYYY-MM-DD). table_name (str): A table to update (corresponding query should be placed in the queries/ folder of the flow). sql_loader (SQLLoader): An object of SQLLoader class which tied to a correct path to /queries folder for a specific ETL. kwargs (dict): Custom arguments. """ sql_template = sql_loader.load_query('update_{}'.format(table_name)) sql, non_identifier_params = self.validator.format_identifiers( sql_template, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], date=date, storeid=self.storeid, feedid=self.feedid, **kwargs)) return self.fetchone( sql, params=non_identifier_params, dict_cursor=True)