from ...base import BaseStep from ...mixins import ESMixin, SnowflakeMixin from ...search.constants import ARTIST_SEARCH_INDEX_NAME from ...utils import Cleanup from ..constants import OK_STATUS, VALIDATION_ES_ARTIST_INDEX __all__ = ["Validation"] class Validation(SnowflakeMixin, ESMixin, BaseStep): depends_on = {Cleanup} base_index_name: str = ARTIST_SEARCH_INDEX_NAME @property def index_name(self) -> str: return f"{self.base_index_name}_{self.timestamp}" def _get_total_artist_count_es(self) -> int: body: dict = {"query": {"match_all": {}}} result = self.es_client.count(index=self.index_name, body=body) return result["count"] def _get_total_artist_count_sf(self): query = """ SELECT COUNT(*) AS COUNTER FROM ( SELECT DISTINCT T_NON_SIGNED_ARTIST.ID FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST AS T_NON_SIGNED_ARTIST ) """ result = self.run_raw_query(query=query, is_async=False) result = result[0].get("counter") # type: ignore return result def _check_es_artist_search_index(self) -> str: status = OK_STATUS res: int = self._get_total_artist_count_es() if not res: status = VALIDATION_ES_ARTIST_INDEX["es_artist_search_index"] return status def _check_number_es_index_and_sf_non_signed_artists(self) -> str: """ Quantity of artists in SF T_NON_SIGNED_ARTIST should be the same to ES artists search index """ status = OK_STATUS sf_count = self._get_total_artist_count_sf() es_count = self._get_total_artist_count_es() if sf_count != es_count: status = VALIDATION_ES_ARTIST_INDEX["compare_es_artist_index_with_sf_non_signed_artist"] status = status + f": ES count {es_count}, SF count {sf_count}" return status def process(self): result = { "es_artist_search_index": self._check_es_artist_search_index(), "compare_es_artist_index_with_sf_non_signed_artist": self._check_number_es_index_and_sf_non_signed_artists(), } if all(x == OK_STATUS for x in result.values()): self.logger.info(result) else: self.logger.error(result) return result