from datetime import date, timedelta from src import config, logger from src.clients import clients from src.constants import PLAYLIST_DIGEST_STREAMS_COUNTY_CODES, Action, ApolloKey, DataType, PlaylistDigestMetric from src.constants.queries.mysql.playlists_digest import DESTINATION_TABLE_PLAYLIST_DIGEST_METRICS, \ DOWNLOAD_COLUMNS_PLAYLIST_DIGEST_METRICS from src.constants.queries.snowflake.playlists_digest import AGGREGATE_PLAYLIST_DIGEST_METRICS, GET_LATEST_DATE from src.processors.base import BaseProcessor class PlaylistDigestMetricsProcessor(BaseProcessor): _data_type = DataType.PLAYLIST_DIGEST_METRICS _save_temp_table = False _snowflake_query = AGGREGATE_PLAYLIST_DIGEST_METRICS _destination_table = DESTINATION_TABLE_PLAYLIST_DIGEST_METRICS _destination_columns = DOWNLOAD_COLUMNS_PLAYLIST_DIGEST_METRICS @property def database(self): if self._metric == PlaylistDigestMetric.STREAMS: return config.Snowflake.EXPLORATION_DATABASE return config.Snowflake.PUBLIC_DATA_MAIN_DATABASE @property def schema(self): if self._metric == PlaylistDigestMetric.STREAMS: return config.Snowflake.EXPLORATION_EXP_SCHEMA return config.Snowflake.PUBLIC_DATA_MAIN_PLAYLISTS_SCHEMA @property def snowflake_query(self) -> str: query = self._snowflake_query[self._metric] return query.format( database=self.database, schema=self.schema, date_from=self._previous_date, days_count=self._days_count, min_value=config.METRIC_MIN_VALUE[self._metric], **( dict( start_date=self._previous_date - timedelta(days=13), end_date=self._latest_date, country_code_list=",".join(f"'{i}'" for i in PLAYLIST_DIGEST_STREAMS_COUNTY_CODES), ) if self._metric == PlaylistDigestMetric.STREAMS else {} ), ) @property def destination_table(self) -> str: return self._destination_table[self._metric] @property def destination_columns(self) -> str: return self._destination_columns[self._metric] def parse_type_specific_data(self): self._metric = self._payload["metric"] if self._metric not in (PlaylistDigestMetric.STREAMS, PlaylistDigestMetric.FOLLOWERS): raise ValueError(f"Invalid metric {self._metric}") data = self._payload.get("data", {}) self._latest_date, self._previous_date = data.get("latest_date"), data.get("previous_date") if Action.is_after(self._action, Action.SNOWFLAKE_EXECUTE): if not self._latest_date: raise ValueError(f"Invalid latest date {self._latest_date}") if not self._previous_date: raise ValueError(f"Invalid previous date {self._previous_date}") self._latest_date, self._previous_date = ( date.fromisoformat(self._latest_date), date.fromisoformat(self._previous_date) ) def get_latest_date(self) -> date: query = GET_LATEST_DATE[self._metric].format(database=self.database, schema=self.schema) self._latest_date = clients.snowflake.get_single_value(query) logger.log.info(f"{self._run_id}: {self._metric} SF last date {self._latest_date}") return self._latest_date def get_previous_date(self, default_date: date) -> date: self._previous_date = clients.mysql.get_date_value( ApolloKey.PLAYLIST_DIGEST_METRIC_DATE[self._metric], default_date ) logger.log.info(f"{self._run_id}: {self._metric} MySQL last date {self._previous_date}") if not config.OVERRIDE_PREVIOUS_DATE: self._previous_date = self._previous_date + timedelta(days=1) return self._previous_date def pre_snowflake_query(self): self.get_latest_date() self.get_previous_date(self._latest_date) self._days_count = (self._latest_date - self._previous_date).days + 1 if not self._days_count: raise ValueError(f"Nothing to update {self._latest_date}") max_days_count = config.MAX_DAYS_COUNT[self._metric] if self._days_count > max_days_count: self._days_count = max_days_count self._latest_date = self._previous_date + timedelta(days=max_days_count) self._result.update( {"latest_date": self._latest_date.isoformat(), "previous_date": self._previous_date.isoformat()} ) logger.log.info( f"{self._run_id}: {self._metric} updating {self._previous_date}-{self._latest_date} ({self._days_count})" ) def pre_mysql_query(self): super().pre_mysql_query() if config.OVERRIDE_PREVIOUS_DATE: logger.log.info(f"{self._run_id}: {self._metric} removing date {self._previous_date}") clients.mysql.delete_date_range(self.destination_table, self._previous_date, self._previous_date) @property def other_metric(self): if self._metric == PlaylistDigestMetric.STREAMS: return PlaylistDigestMetric.FOLLOWERS return PlaylistDigestMetric.STREAMS def post_mysql_query(self): super().post_mysql_query() logger.log.info(f"{self._run_id}: {self._metric} updating latest date {self._latest_date}") clients.mysql.set_date_value(ApolloKey.PLAYLIST_DIGEST_METRIC_DATE[self._metric], self._latest_date) logger.log.info(f"{self._run_id}: {self.other_metric} getting latest date") other_metric_latest_date = clients.mysql.get_date_value( ApolloKey.PLAYLIST_DIGEST_METRIC_DATE[self.other_metric] ) total_latest_date = ( self._latest_date if not other_metric_latest_date or other_metric_latest_date > self._latest_date else other_metric_latest_date ) logger.log.info(f"{self._run_id}: updating common latest date {total_latest_date}") clients.mysql.set_date_value(ApolloKey.PLAYLIST_DIGEST_LATEST_DATE, total_latest_date)