"""Snowflake connector class for the Spotify's tasks.""" import boto3 from snowflake_connector.etl_connector import SnowflakeSQLExecutor from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.flows.itunes import config from feed_ingestion.util.snowflake.errors import JSONParserLoading sql_loader = SQLLoader(__file__) class ITunes(SnowflakeSQLExecutor): """Helper class to abstract Snowflake operations. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate some flow specific operations. """ @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed. Returns: str: Feed name """ return config.feed_name @property def feedid(self): """Id of the feed. Returns: str: Feed id """ return config.feedid @property def storeid(self): """Storeid of feed data. Should match dim_store and feed config value. Returns: integer: Feed's storeid. """ return config.storeid def load_temp_staging_raw_table( self, temp_staging_raw_table, aws, key_dir, **kwargs): """Load temp staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. aws (dict): [DEPRECATED] AWS credentials to fill a template of COPY SQL statement. key_dir (str): A S3 path to load files from. """ credentials = boto3.Session().get_credentials() aws_key_id = credentials.access_key aws_secret_key = credentials.secret_key aws_token = credentials.token if credentials.token else '' if kwargs.get('error_on_column_count_mismatch', '').lower() == 'false': error_on_column_count_mismatch = 'FALSE' else: error_on_column_count_mismatch = 'TRUE' params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table, s3_path=key_dir, aws_key_id=aws_key_id, aws_secret_key=aws_secret_key, aws_token=aws_token, on_error_action='SKIP_FILE_{}'.format(kwargs['error_limit']), error_on_column_count_mismatch=error_on_column_count_mismatch,) return [JSONParserLoading(*e) for e in self.fetchall_query( sql_loader, kwargs['query_name'], params)]