"""Snowflake connector class for the Apple Music Streams' tasks.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.staging_raw_sf.base_executor import \ SnowflakeSQLExecutorSR from feed_ingestion.flows.music_analytics_reports import config sql_loader = SQLLoader(__file__) class MusicAnalyticsReportsSE(SnowflakeSQLExecutorSR): """Helper class to abstract Snowflake operations. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate some flow specific operations. """ @property def staging_raw_table(self): """Name of the staging_raw table for the feed. Returns: str: staging_raw_{feed} table name. """ return config.snowflake_tables['staging_raw'] @property def errors_table(self): """Name of the errors table for the feed. Returns: str: {feed}_errors table name. """ return config.snowflake_tables['errors'] def create_temp_staging_raw_table(self, temp_staging_raw_table, **kwargs): """Create a temporary staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. kwargs (dict): Custom arguments with report type. """ report_type = temp_staging_raw_table.replace( config.snowflake_tables['temp_staging'].split('{')[0], '')[:-9] report_loader = SQLLoader(__file__, folder=f'/queries/{report_type}') if 'errors' in temp_staging_raw_table: query_name = 'create_temp_' + self.errors_table.format( report_type=report_type) else: query_name = 'create_temp_' + self.staging_raw_table.format( report_type=report_type) params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table) self.execute_query(report_loader, query_name, params) def load_temp_staging_raw_table( self, temp_staging_raw_table, aws, key_dir: dict, **kwargs): """Load temp staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. aws (dict): AWS credentials to fill a template of COPY SQL statement. key_dir (str): Custom arguments. """ query_name = 'load_temp_music_analytics_report' s3_path = key_dir.get('errors') if \ 'errors' in temp_staging_raw_table else key_dir.get('report') aws_params = self.get_aws_params() params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table, s3_path=s3_path, **aws_params ) return self.execute_query(sql_loader, query_name, params) def load_staging_raw_table( self, temp_staging_raw_table, staging_raw_table, date, **kwargs): """Create a temporary staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. staging_raw_table (str): A staging table name in Snowflake. kwargs (dict): Custom arguments with report type. """ report_type = staging_raw_table.replace( config.snowflake_tables['staging_raw'].split('{')[0], '') report_loader = SQLLoader(__file__, folder=f'/queries/{report_type}') query_name = 'load_' + staging_raw_table filename = kwargs.get('errors') if \ 'errors' in staging_raw_table else kwargs.get('report') params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], staging_fact_table=staging_raw_table, temp_staging_fact_table=temp_staging_raw_table, download_date=date, filename=filename ) self.execute_query(report_loader, query_name, params)