"""Snowflake connector class for the deezer's tasks.""" from datetime import datetime from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.fact_analytics_sf.base_executor \ import SnowflakeSQLExecutorFA, sql_loader as base_sql_loader from feed_ingestion.common.staging_raw_sf.base_executor \ import SnowflakeSQLExecutorSR from feed_ingestion.flows.deezer import config from feed_ingestion.flows.deezer.config import get_version # Load SQL templates sql_loader = SQLLoader(__file__) class DeezerBase(SnowflakeSQLExecutorFA, SnowflakeSQLExecutorSR): """Helper class to abstract Snowflake operations. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate some flow specific operations. """ @property def licensor(self): """Licensor value.""" # Shall be implemented in derived class raise NotImplementedError() @property def feed_name(self): """Name of the feed. Returns: str: Feed name, e.g. 'deezer_daily_theorchard'. """ return '_'.join([config.feed_name, self.licensor]) @property def feedid(self): """Id of the feed. Returns: int: Feed id """ return config.feedid @property def storeid(self): """Storeid of feed data. Should match dim_store and feed config value. Returns: integer: Feed's storeid. """ return 348 def staging_raw_table(self, spec_version=3): """Name of the staging_raw table for the feed. Returns: str: staging_raw_{feed} table name. """ if spec_version == 3: return f'STAGING_RAW_DEEZER_V3_COMBINED_{self.licensor.upper()}' else: return config.snowflake_table_names['staging_raw'][ config.names_alias[f'{self.licensor}_v{spec_version}'] ] def _delete_from_fact_table(self, table, date, **kwargs): """Delete rows in fact table with the current run date. This is required for the workflow to be idempotent, and to avoid row duplication. Before we'll load rows for a specific day to the fact tables, we have to delete rows which were added by previous (allegedly unsuccessful workflow run). Args: table (str): Either fact_analytics, either fact_analytics_error. date (str): Date of the data being process (YYYY-MM-DD). kwargs (dict): Custom arguments. """ sql_template = sql_loader.load_query('delete_from_fact_table') params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_table=table, reportdate=date, storeid=self.storeid, feedid=self.feedid) params.update(kwargs) sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) return self.fetchone(sql_template, params=non_identifier_params)[0] def load_staging_fact_table(self, date): """Load staging fact_analytics table from staging_raw table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ spec_version = get_version( config.spec_version[self.licensor], datetime.strptime(date, '%Y-%m-%d').date() ) query_name = 'load_staging_fact_{}'.format(self.licensor) # views separated by licensor used for v3 fact load, so same queries if spec_version == 3: query_name = 'load_staging_fact_all_v3' self.execute_query( sql_loader, query_name, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], staging_fact_table=self.staging_fact_table(date), staging_raw_table=self.staging_raw_table(spec_version), reportdate=date, storeid=self.storeid, feedid=self.feedid)) def load_fact_data(self, date, **kwargs): """Load matched data into fact_analytics. This method is pretty generic, so it's implemented in base FA executor class. Args: date (str): Date of the data being process (YYYY-MM-DD). """ spec_version = get_version( config.spec_version[self.licensor], datetime.strptime(date, '%Y-%m-%d').date() ) sql_template = base_sql_loader.load_query('load_fact_analytics') params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_table=self.fact_table, staging_fact_analytics_table=self.staging_fact_table(date), staging_raw_table=self.staging_raw_table(spec_version), reportdate=date) sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) self.execute(sql_template, params=non_identifier_params) def load_fact_error_data(self, date, **kwargs): """Load unmatched data into fact_analytics_error. Args: date (str): Date of the data being process (YYYY-MM-DD). """ spec_version = get_version( config.spec_version[self.licensor], datetime.strptime(date, '%Y-%m-%d').date() ) # views separated by licensor used for v3 fact load, so same queries if spec_version < 3: query_name = \ f'load_fact_analytics_error_{self.licensor}_v{spec_version}' else: query_name = 'load_fact_analytics_error_all_v3' self.execute_query( sql_loader, query_name, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_error_table=self.fact_error_table, staging_fact_table=self.staging_fact_table(date), staging_raw_table=self.staging_raw_table(spec_version), reportdate=date, storeid=self.storeid, feedid=self.feedid)) def update_dimension_table(self, date, table_name): """Update dimension table with the new data. Args: date (str): Date of the data being process (YYYY-MM-DD). table_name (str): A table to update (corresponding query should be placed in the queries/ folder of the flow). """ spec_version = get_version( config.spec_version[self.licensor], datetime.strptime(date, '%Y-%m-%d').date() ) if spec_version == 3: table_name += '_v3' return self._update_dimension_table( date, table_name, sql_loader, licensor=self.licensor) class DeezerTheOrchard(DeezerBase): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'theorchard' class DeezerSme(DeezerBase): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'sme' class DeezerAltafonte(DeezerBase): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'altafonte'