"""Snowflake connector class for the YouTube Facts tasks.""" from snowflake_connector.etl_connector import SQLLoader from feed_ingestion.common.fact_analytics_sf.base_executor import \ SnowflakeSQLExecutorFA from feed_ingestion.common.staging_raw_sf.base_executor import \ SnowflakeSQLExecutorSR from feed_ingestion.flows.youtube_facts import config sql_loader = SQLLoader(__file__) class YouTubeFacts(SnowflakeSQLExecutorFA, SnowflakeSQLExecutorSR): """Helper class to abstract Snowflake operations. This class inherits from SnowflakeSQLExecutor class, which provides basic set of methods. This class extends SnowflakeSQLExecutor with some specific methods, which are useful to encapsulate some flow specific operations. """ @property def feedid(self): """Id of the feed. Returns: str: Feed id """ return config.feedid @property def storeid(self): """Storeid of feed data. Should match dim_store and feed config value. Returns: integer: Feed's storeid. """ return config.storeid @property def staging_raw_table(self): """Stub for staging raw table name. Returns: str: staging_raw_{feed} table name. """ return '' @property def report_type(self): """Type of reports to ingest (asset or video). Returns str: Report type. """ raise NotImplementedError() @property def licensor(self): """Licensor to ingest (one of config.licensor). Returns str: Report type. """ raise NotImplementedError() @property def feed_name(self): """Name of the feed. Should match dir name of this feed, feed_name in config.py of a feed Returns: str: Feed name """ return 'youtube_facts_{}_{}'.format(self.report_type, self.licensor) @property def fact_table(self): """Name of the fact analytics table for feed. Returns: str: Fact analytics table name. """ return 'fact_youtube_{}_analytics'.format(self.report_type) @property def fact_error_table(self): """Name of the fact analytics error table for feed. Returns: str: Fact analytics error table name. """ return 'fact_youtube_{}_analytics_error'.format(self.report_type) @property def demographics_table(self): """Name of the demographics table for feed. Returns: str: Demographics table name. """ return 'fact_youtube_{}_demographics'.format(self.report_type) def load_staging_fact_table(self, date): """Load staging fact_analytics table from staging_raw table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ query_name = 'load_staging_fact_{}'.format(self.report_type) self.execute_query( sql_loader, query_name, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], staging_fact_analytics_table=self.staging_fact_table(date), reportdate=date, storeid=self.storeid, feedid=self.feedid, staging_db=config.snowflake['staging_db'], staging_schema=config.snowflake['staging_schema'], licensor=self.licensor)) def delete_from_demographics_table(self, date, **kwargs): """Delete rows in demographics table with the current run date. Args: date (str): Date of the data being process (YYYY-MM-DD). kwargs (dict): Custom arguments. """ self._delete_from_fact_table(self.demographics_table, date, **kwargs) def _delete_from_fact_table(self, table, date, **kwargs): """Delete rows in fact table with the current run date. This is required for the workflow to be idempotent, and to avoid row duplication. Before we'll load rows for a specific day to the fact tables, we have to delete rows which were added by previous (allegedly unsuccessful workflow run). Args: table (str): Either fact_analytics, either fact_analytics_error. date (str): Date of the data being process (YYYY-MM-DD). kwargs (dict): Custom arguments. """ sql_template = sql_loader.load_query('delete_from_fact_table') params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_table=table, reportdate=date, storeid=self.storeid, feedid=self.feedid, **kwargs) sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) return self.fetchone(sql_template, params=non_identifier_params)[0] def load_fact_data(self, date, **kwargs): """Load matched data into fact table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ query_name = 'load_fact_analytics_{}'.format(self.report_type) sql_template = sql_loader.load_query(query_name) params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_table=self.fact_table, staging_fact_analytics_table=self.staging_fact_table(date), reportdate=date, **kwargs) sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) self.execute(sql_template, params=non_identifier_params) def load_fact_error_data(self, date, **kwargs): """Load matched data into fact error table. Args: date (str): Date of the data being process (YYYY-MM-DD). """ query_name = 'load_fact_analytics_error_{}'.format(self.report_type) sql_template = sql_loader.load_query(query_name) params = dict( db=self.sf_config['db'], schema=self.sf_config['schema'], fact_error_table=self.fact_error_table, staging_fact_analytics_table=self.staging_fact_table(date), reportdate=date, **kwargs) sql_template, non_identifier_params = ( self.validator.format_identifiers(sql_template, params)) self.execute(sql_template, params=non_identifier_params) def load_demographics_data(self, date, **kwargs): """Load demographics data. Args: date (str): Date of the data being process (YYYY-MM-DD). """ query_name = 'load_fact_demographics_{}'.format(self.report_type) self.execute_query( sql_loader, query_name, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], demographics_table=self.demographics_table, reportdate=date, storeid=self.storeid, feedid=self.feedid, staging_db=config.snowflake['staging_db'], staging_schema=config.snowflake['staging_schema'], **kwargs)) def load_youtube_video_asset_type_mapping(self, date): """Load youtube_video_asset_type_mapping data. Args: date (str): Date of the data being process (YYYY-MM-DD). """ query_name = 'load_youtube_video_asset_type_mapping' self.execute_query( sql_loader, query_name, params=dict( db=self.sf_config['db'], schema=self.sf_config['schema'], reportdate=date, staging_fact_analytics_table=self.staging_fact_table(date))) class SmeLicensorMixin: """Licensor mixin.""" @property def licensor(self): """Licensor value.""" return 'sme' class OrchardLicensorMixin: """Licensor mixin.""" @property def licensor(self): """Licensor value.""" return 'theorchard' class YouTubeFactsVideo(YouTubeFacts): """Snowflake executor for YouTube Video report type.""" @property def report_type(self): """Type of reports to ingest (asset or video). Returns str: Report type. """ return 'video' def load_fact_error_data(self, date, **kwargs): """Load matched data into fact error table. Video report type doesn't use error table. Skip this step. Args: date (str): Date of the data being process (YYYY-MM-DD). """ return None def delete_from_fact_error_table(self, date, **kwargs): """Delete data into fact error table. Video report type doesn't use error table. Skip this step Args: date (str): Date of the data being process (YYYY-MM-DD). """ return None class YouTubeFactsAsset(YouTubeFacts): """Snowflake executor for YouTube Asset report type.""" @property def report_type(self): """Type of reports to ingest (asset or video). Returns str: Report type. """ return 'asset' class YouTubeFactsVideoTheOrchard(OrchardLicensorMixin, YouTubeFactsVideo): """Executor for specific licensor and report.""" pass class YouTubeFactsAssetTheOrchard(OrchardLicensorMixin, YouTubeFactsAsset): """Executor for specific licensor and report.""" pass class YouTubeFactsVideoSme(SmeLicensorMixin, YouTubeFactsVideo): """Executor for specific licensor and report.""" pass class YouTubeFactsAssetSme(SmeLicensorMixin, YouTubeFactsAsset): """Executor for specific licensor and report.""" pass