"""Snowflake connector class for MRC tasks.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.staging_raw_sf.base_executor \ import SnowflakeSQLExecutorSR from feed_ingestion.flows.mrc import config # Load SQL templates sql_loader = SQLLoader(__file__) mapping_sql_loader = SQLLoader(__file__, folder='/mapping_queries') class MRCSFExecutor(SnowflakeSQLExecutorSR): """Helper class to abstract Snowflake operations.""" @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed, and a key of executors dict in feed_ingestion/common/ fact_analytics_sf/__init__.py file. Returns: str: Feed name, e.g. 'amazon_unlimited'. """ return config.feed_name @property def staging_raw_table(self): """Name of the staging_raw table for the feed. Returns: str: staging_raw_{feed} table name. """ return config.snowflake_table_names['staging_raw'] def create_temp_staging_raw_table(self, temp_staging_raw_table, **kwargs): """Create a temporary staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. kwargs (dict): Custom arguments with date. """ loader = sql_loader if 'raw_mrc_' + kwargs.get('download_date') in \ temp_staging_raw_table else mapping_sql_loader params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table) self.execute_query(loader, 'create_' + temp_staging_raw_table. replace('_mrc', ''). replace('_' + kwargs.get('download_date'), ''), params) def load_temp_staging_raw_table( self, temp_staging_raw_table, aws, key_dir, **kwargs): """Load temp staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. aws (dict): AWS credentials to fill a template of COPY SQL statement. key_dir (dict): Custom arguments with date. """ loader = sql_loader if 'raw_mrc_' + kwargs.get('download_date') in \ temp_staging_raw_table else mapping_sql_loader params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table, s3_path=key_dir, aws_key_id=aws['access_key'], aws_secret_key=aws['access_secret']) self.execute_query(loader, 'load_temp_staging_raw', params) def load_staging_raw_table( self, temp_staging_raw_table, staging_raw_table, date, **kwargs): """Load the temp_staging_raw data to the feed's staging_raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ loader = sql_loader if 'raw_mrc_' + date.replace('-', '') in \ temp_staging_raw_table else mapping_sql_loader params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table, staging_raw_table=staging_raw_table, filename=kwargs.get('file'), download_date=kwargs.get('download_date')) self.execute_query(loader, 'load_staging_raw', params) def select_remaining_tables( self, temp_table_pattern, date): """Select remaining temp_staging_raw tables. Args: temp_table_pattern (str): A table name pattern in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ pattern = f'%{temp_table_pattern.upper()}' \ f"%_{date.replace('-', '')}%" # e.g. '%TEMP_STAGING_RAW_MRC%_20230129%' params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_table_pattern=pattern) return self.fetchall_query( sql_loader, 'select_remaining_tables', params)