"""Snowflake connector class for the tasks of the YouTube claim workflow.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.fact_analytics_sf.base_executor \ import SnowflakeSQLExecutorFA from feed_ingestion.common.youtube_sf.base_executor import YouTubeExecutor from feed_ingestion.flows.youtube_claim import config # Load SQL templates sql_loader = SQLLoader(__file__) class YoutubeClaimExecutor(SnowflakeSQLExecutorFA, YouTubeExecutor): """Helper class to abstract Snowflake operations. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate some flow specific operations. """ @property def licensor(self): """Licensor to ingest (one of config.licensor). Returns str: Report type. """ raise NotImplementedError() @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed, and a key of executors dict in feed_ingestion/common/ fact_analytics_sf/__init__.py file. Returns: str: Feed name, e.g. 'deezer_daily_sf'. """ return '_'.join([config.feed_name, self.licensor]) @property def feedid(self): """Id of the feed. Returns: int: Feed id """ return config.feedid @property def storeid(self): """Storeid of feed data. Should match dim_store and feed config value. Returns: integer: Feed's storeid. """ return config.storeid @property def staging_raw_table(self): """Name of the staging_raw table for the feed. Returns: str: staging_raw_{feed} table name. """ return config.snowflake_table_names[f'staging_raw_{self.licensor}'] @property def channel_owner(self): """Name of the channel owner. Returns: str: channel_owner. """ return 'UGC' def create_temp_staging_raw_table(self, temp_staging_raw_table): """Create a temporary staging raw table for youtube claim data. Args: temp_staging_raw_table (str): A table name in Snowflake. """ self.execute_query( sql_loader, f'create_temp_staging_raw_{self.licensor}', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table)) def load_temp_staging_raw_table(self, temp_staging_raw_table, key_dir): """Load temp staging raw table with a Pandora raw file. Args: temp_staging_raw_table (str): A table name in Snowflake. key_dir (str): A S3 path to load files from. """ aws_params = self.get_aws_params() self.execute_query( sql_loader, 'load_temp_staging_raw', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], temp_staging_raw_table=temp_staging_raw_table, s3_path=key_dir, **aws_params ) ) def clean_staging_raw_table(self, staging_raw_table, date): """Delete rows from previous unsuccessful workflow run. Args: staging_raw_table (str): A table name in Snowflake. date (str): Date of the data being process (YYYY-MM-DD). """ self.execute_query( sql_loader, f'delete_from_staging_raw_{self.licensor}', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], staging_raw_table=staging_raw_table, date=date, licensor=self.licensor)) def load_staging_raw_table( self, temp_staging_raw_table, staging_raw_table, date, processed_datetime=None, filename=None, filesize=None): """Load the temp_staging_raw data to the feed's staging_raw table. Args: date (str): Date of the data being process (YYYY-MM-DD). processed_datetime (str): A single processed daytime to use through all the tables during the workflow run. staging_raw_table (str): A table name in Snowflake. temp_staging_raw_table (str): Name of the temp table. filename (str): Filename of the streams data from which a particular temp streams table was loaded. filesize (int): Size of the file in bytes. """ self.execute_query( sql_loader, f'load_staging_raw_{self.licensor}', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], filename=filename, filesize=filesize, download_date=date, temp_table_name=temp_staging_raw_table, processed_datetime=processed_datetime, staging_raw_table=staging_raw_table, licensor=self.licensor)) def load_staging_fact_table(self, date): """Load staging fact_analytics table from staging_raw table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ query_name = 'load_staging_fact' self.execute_query( sql_loader, query_name, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], staging_fact_table=self.staging_fact_table(date), staging_raw_table=self.staging_raw_table, reportdate=date, storeid=self.storeid, feedid=self.feedid)) def load_fact_error_data(self, date): """Load unmatched data into fact_analytics_error. Args: date (str): Date of the data being process (YYYY-MM-DD). """ self.execute_query( sql_loader, 'load_fact_analytics_error', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_error_table=self.fact_error_table, staging_fact_table=self.staging_fact_table(date), staging_raw_table=self.staging_raw_table, reportdate=date, storeid=self.storeid, feedid=self.feedid, licensor=self.licensor)) def is_assets_available(self): """Check if staging_raw_youtube_asset_report table is not empty.""" return self.fetchone_query( sql_loader, 'check_assets_availability', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], assets_table='staging_raw_youtube_asset_report', licensor=self.licensor))[0] def update_dim_claim(self, date): """Update dim_claim table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ return self.execute_query( sql_loader, 'update_dim_claim', params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], download_date=date, staging_raw_table=self.staging_raw_table, licensor=self.licensor)) class YoutubeClaimTheOrchard(YoutubeClaimExecutor): """Snowflake executor for specific licensor.""" @property def licensor(self): """Licensor value.""" return 'theorchard' class YoutubeClaimSME(YoutubeClaimExecutor): """Snowflake executor for specific licensor.""" @property def licensor(self): """Licensor value.""" return 'sme'