"""Snowflake connector class for the TikTok and Douyin tasks.""" from datetime import datetime from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.staging_raw_sf.base_executor \ import SnowflakeSQLExecutorSR from feed_ingestion.flows.tiktok import config from feed_ingestion.util.snowflake.errors import JSONParserLoading sql_loader = SQLLoader(__file__) class TikTok(SnowflakeSQLExecutorSR): """Helper class to abstract Snowflake operations. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate some flow specific operations. """ @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed. Returns: str: Feed name """ return config.feed_name @property def licensor(self): """Licensor name. Returns str: Licensor name. """ raise NotImplementedError() @property def query_name_suffix(self): """Licensor name. Returns str: Licensor name. """ raise NotImplementedError() def get_query_name_suffix(self, date): """Create default implementation for all TikTok subclasses. Args: date (str): date param from feed execution context """ return self.query_name_suffix def create_temp_staging_raw_table(self, temp_staging_raw_table, **kwargs): """Create a temporary staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. """ sql_loader = SQLLoader(__file__, date=kwargs['download_date']) query_suffix = self.get_query_name_suffix(kwargs['download_date']) query = f'create_temp_staging_raw_{query_suffix}' params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table) self.execute_query(sql_loader, query, params) def load_temp_staging_raw_table( self, temp_staging_raw_table, aws, key_dir, **kwargs): """Load temp staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. aws (dict): AWS credentials to fill a template of COPY SQL statement. key_dir (str): A S3 path to load files from. """ aws_params = self.get_aws_params() query_suffix = self.get_query_name_suffix(kwargs['download_date']) query = f'load_temp_staging_raw_{query_suffix}' params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table, s3_path=key_dir, on_error_action='SKIP_FILE_{}'.format(kwargs['error_limit']), **aws_params) return [JSONParserLoading(*e) for e in self.fetchall_query( sql_loader, query, params)] def clean_staging_raw_table(self, staging_raw_table, date, **kwargs): """Delete rows from previous unsuccessful workflow run. Args: staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ query = 'delete_from_staging_raw' params = dict( consumer_db=config.consumer_sf['db'], consumer_schema=config.consumer_sf['schema'], staging_raw_table=staging_raw_table, date=date, platform=kwargs['report_name'], licensor=self.licensor) self.execute_query(sql_loader, query, params) def load_staging_raw_table( self, temp_staging_raw_table, staging_raw_table, date, **kwargs): """Load the temp_staging_raw data to the feed's staging_raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ sql_loader = SQLLoader(__file__, date=kwargs['download_date']) # generate sql query name query = f'load_staging_raw_{self.get_query_name_suffix(date)}' params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], consumer_db=config.consumer_sf['db'], consumer_schema=config.consumer_sf['schema'], download_date=date, temp_table_name=temp_staging_raw_table, staging_raw_table=staging_raw_table, licensor=self.licensor) self.execute_query(sql_loader, query, params) def update_staging_raw_table( self, staging_raw_table, date, **kwargs): """Load the temp_staging_raw data to the feed's staging_raw table. Args: staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ sql_loader = SQLLoader(__file__, date=date) # generate sql query name query = 'remap_fingerprint_isrc_product_code' params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], consumer_db=config.consumer_sf['db'], consumer_schema=config.consumer_sf['schema'], download_date=date, staging_raw_table=staging_raw_table) self.execute_query(sql_loader, query, params) class TikTokTheOrchard(TikTok): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'theorchard' @property def query_name_suffix(self): """Query name suffix.""" return 'theorchard' class TikTokAltafonte(TikTokTheOrchard): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'altafonte' @property def query_name_suffix(self): """Query name suffix.""" return 'theorchard' # reuse theorchard queries def get_query_name_suffix(self, date): """Return different suffix based on execution date. Args: date (str): date param from feed execution context """ if (datetime.strptime(date, '%Y-%m-%d') >= datetime.strptime( config.altafonte_source_bucket_switch_date, '%Y-%m-%d')): # now orchard suffix is fine, but it might change in the future return 'theorchard' else: return self.query_name_suffix class TikTokSME(TikTok): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'sme' @property def query_name_suffix(self): """Query name suffix.""" return 'sme' class TikTokSMEJP(TikTokSME): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'smejp' @property def query_name_suffix(self): """Query name suffix.""" return 'sme' # reuse SME queries class TikTokSMEJPIntl(TikTokSME): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor.""" return 'smejpintl' @property def query_name_suffix(self): """Query name suffix.""" return 'sme' # reuse SME queries