"""Snowflake connector class for the YouTube Channels tasks.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.staging_raw_sf.base_executor import \ SnowflakeSQLExecutorSR from feed_ingestion.flows.youtube_channel_names import config sql_loader = SQLLoader(__file__) class YouTubeChannelNames(SnowflakeSQLExecutorSR): """Helper class to abstract Snowflake operations.""" @property def licensor(self): """Licensor to ingest (one of config.licensor). Returns str: Report type. """ raise NotImplementedError() @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed. Returns: str: Feed name """ return '_'.join([config.feed_name, self.licensor]) @property def dim_table(self): """Name of YouTube channel names dim table. Returns: str: Dim table name """ return config.snowflake['dim_table'] def create_temp_staging_raw_table(self, temp_staging_raw_table): """Create a temporary staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. """ params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table, dim_table=self.dim_table) self.execute_query(sql_loader, 'create_temp_staging_raw', params) def load_temp_staging_raw_table( self, temp_staging_raw_table, aws, key_dir, **kwargs): """Load temp staging raw table. Args: temp_staging_raw_table (str): A table name in Snowflake. aws (dict): Deprecated AWS credentials to fill a template of COPY SQL statement. key_dir (str): A S3 path to load files from. """ aws_params = self.get_aws_params() params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], table_name=temp_staging_raw_table, s3_path=key_dir, file_pattern=kwargs['file_pattern'], **aws_params) self.fetchall_query(sql_loader, 'load_temp_staging_raw', params) def get_missing_channels(self, date): """Fetch channel ids missing from mapping table. Args: date (str): Reporting date (YYYY-MM-DD). Returns: list: List of missing channel ids """ params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], date=date, dim_table=self.dim_table, licensor=self.licensor) sql_template = sql_loader.load_query('get_missing_channels') sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) res = self.fetchall(sql_template, params=non_identifier_params) return res def update_dim_table( self, s3_path, file_name, temp_staging_raw_table): """Update channel names mapping table. Args: s3_path (str): Path to archive directory. file_name (str): Preprocessed file name. temp_staging_raw_table (str): A table name in Snowflake. """ aws_params = self.get_aws_params() params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], dim_table=self.dim_table, temp_staging_raw_table=temp_staging_raw_table, **aws_params) self.execute_query(sql_loader, 'update_dim_table', params) class YouTubeChannelNamesTheOrchard(YouTubeChannelNames): """Snowflake executor for specific licensor.""" @property def licensor(self): """Licensor value.""" return 'theorchard' class YouTubeChannelNamesSME(YouTubeChannelNames): """Snowflake executor for specific licensor.""" @property def licensor(self): """Licensor value.""" return 'sme'