"""Contains reloaded SnowflakeExecutor for spotify_artificial_streams flow.""" __all__ = [ 'SpotifyArtificialStreamsSnowflakeSQLExecutor', ] from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.staging_raw_sf import base_executor from feed_ingestion.flows.spotify_artificial_streams import config from feed_ingestion.util.snowflake.errors import JSONParserLoading class SpotifyArtificialStreamsSnowflakeSQLExecutor( base_executor.SnowflakeSQLExecutorSR ): """SnowflakeSQLExecutor for spotify_artificial_streams flow.""" sql_loader = SQLLoader(__file__) @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed, and a key of executors dict in feed_ingestion/common/__init__.py file. Returns: str: Feed name, e.g. 'deezer_daily_sf'. """ return config.feed_name @property def staging_raw_table(self): """Name of the staging_raw table for the feed. Returns: str: staging_raw_{feed} table name. """ return config.staging_raw_table_name def create_temp_staging_raw_table( self, temp_staging_raw_table, **kwargs, ): """Create a temporary staging raw table with a datestamp suffix. Args: temp_staging_raw_table (str): A table name in Snowflake. """ params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table, ) self.execute_query(self.sql_loader, 'create_temp_staging_raw', params) def load_temp_staging_raw_table( self, temp_staging_raw_table, aws, key_dir, **kwargs, ): """Load temp staging raw table with raw files. Args: temp_staging_raw_table (str): A table name in Snowflake. aws (Dict[str, str]): AWS credentials to fill a template of COPY SQL statement. [DEPRECATED] key_dir (str): A S3 path to load files from. Returns: List[JSONParserLoading]: list of error descriptions """ file_pattern = kwargs['file_pattern'] aws_params = self.get_aws_params() params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table, s3_path=key_dir, file_pattern=file_pattern, on_error_action='SKIP_FILE_{}'.format( kwargs['snowflake_error_limit'], ), **aws_params ) return [ JSONParserLoading(*e) for e in self.fetchall_query( self.sql_loader, 'load_temp_staging_raw', params, ) ] def clean_staging_raw_on_date( self, staging_raw_table, date, query_file_name, ): """Remove data from staging raw table for given date. Args: staging_raw_table (str): name of staging raw table date (str): date to remove data query_file_name (str): name of the table preparation file. """ params = { 'date': date, 'db': self.sf_config['db'], 'schema': self.sf_config['schema'], 'staging_raw_table': staging_raw_table, } self.execute_query( self.sql_loader, 'clean_staging_raw_on_date', params, ) def load_staging_raw_table( self, temp_staging_raw_table, staging_raw_table, date, **kwargs, ) -> None: """Load the temp_staging_raw data to the feed's staging_raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ params = { 'blob_path': kwargs['blob_path'], 'date': date, 'db': self.sf_config['db'], 'schema': self.sf_config['schema'], 'staging_raw_table': staging_raw_table, 'temp_staging_raw_table': temp_staging_raw_table, } self.execute_query(self.sql_loader, 'load_staging_raw', params)