from typing import Any, Dict from ...base import BaseStep from ...mixins import SnowflakeMixin from ...preparation.non_signed.excluded import EXCLUDED_ARTIST_LIST, STRICT_EXCLUDED_LABELS_LIST from ...utils import Cleanup from ..constants import ALL_DSP_IDS, DSP_TYPE_TIKTOK, OK_STATUS, VALIDATION_STATUS_NON_SIGNED_ARTIST __all__ = ["Validation"] class Validation(SnowflakeMixin, BaseStep): depends_on = {Cleanup} def _check_exist_data_non_signed_artists(self) -> str: """ Do we have any data in table T_NON_SIGNED_ARTIST? """ status = OK_STATUS query = """ SELECT COUNT(*) AS COUNTER FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST """ result_query = self.run_raw_query(query=query, is_async=False) if result_query[0].get("counter") == 0: # type: ignore status = VALIDATION_STATUS_NON_SIGNED_ARTIST["non_signed_artists"] return status def _check_id_non_signed_artist(self) -> str: """ Column ID in table T_NON_SIGNED_ARTIST should have values which contain only integers """ status = OK_STATUS query = """ SELECT T_NON_SIGNED_ARTIST.ID FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST AS T_NON_SIGNED_ARTIST GROUP BY T_NON_SIGNED_ARTIST.ID """ res = self.run_raw_query(query=query, is_async=False) res = [i.get("id") for i in res if not isinstance(i.get("id"), int)] # type: ignore if res: status = VALIDATION_STATUS_NON_SIGNED_ARTIST["artist_ids_type"] status = status + ":" + ",".join(str(x) for x in res) return status def _check_artist_same_label(self) -> str: """ Each artist should have only one label value for each DSP """ status = OK_STATUS query = """ SELECT T_NON_SIGNED_ARTIST.ID AS ID, T_NON_SIGNED_ARTIST.LABEL AS LABEL, COUNT(DISTINCT T_NON_SIGNED_ARTIST.LABEL) AS COUNTER FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST AS T_NON_SIGNED_ARTIST GROUP BY T_NON_SIGNED_ARTIST.ID, T_NON_SIGNED_ARTIST.LABEL HAVING COUNTER > 1 """ res = self.run_raw_query(query=query, is_async=False) # type: ignore if res: status = VALIDATION_STATUS_NON_SIGNED_ARTIST["artists_dsp_label"] self.logger.error(status) status = status + ":" + "\n".join([f"CM_ARTIST_ID {i.get('id')} has few labels" for i in res]) # type: ignore return status def _check_excluded_labels(self) -> str: """ We don't need excluded labels in T_NON_SIGNED_ARTIST """ status = OK_STATUS query = """ SELECT T_NON_SIGNED_ARTIST.LABEL FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST AS T_NON_SIGNED_ARTIST GROUP BY T_NON_SIGNED_ARTIST.LABEL """ res = self.run_raw_query(query=query, is_async=False) list_result_label = [i.get("label") for i in res] # type: ignore strict_label_intersection = list(set(list_result_label) & set(STRICT_EXCLUDED_LABELS_LIST)) if len(strict_label_intersection) > 0: status = VALIDATION_STATUS_NON_SIGNED_ARTIST["excluded_labels"] status = status + ":" + "\n".join(str(x) for x in strict_label_intersection) return status def _check_excluded_artists(self) -> str: """ No excluded artists in T_NON_SIGNED_ARTIST """ status = OK_STATUS query = """ SELECT T_NON_SIGNED_ARTIST.NAME FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST AS T_NON_SIGNED_ARTIST GROUP BY T_NON_SIGNED_ARTIST.NAME """ res = self.run_raw_query(query=query, is_async=False) list_artists = [i.get("name") for i in res] # type: ignore artists = list(set(list_artists) & set(EXCLUDED_ARTIST_LIST)) if len(artists) > 0: status = VALIDATION_STATUS_NON_SIGNED_ARTIST["excluded_artists"] status = status + ":" + ",".join([str(x) for x in artists]) return status def _check_dead_artists(self) -> str: """ We don't need to have dead artists in T_NON_SIGNED_ARTIST """ status = OK_STATUS query = """ SELECT DISTINCT T_NON_SIGNED_ARTIST.ID AS ID FROM DNA.DNA_PUBLIC.T_NON_SIGNED_ARTIST AS T_NON_SIGNED_ARTIST JOIN DELPHI_EXPLORATION.CHARTMETRIC.V_CM_ARTIST AS V_CM_ARTIST ON T_NON_SIGNED_ARTIST.ID = V_CM_ARTIST.ID WHERE V_CM_ARTIST.DATE_OF_DEATH IS NOT NULL """ res = self.run_raw_query(query=query, is_async=False) if res: status = VALIDATION_STATUS_NON_SIGNED_ARTIST["dead_artists"] artist_ids = [i.get("id") for i in res] # type: ignore self.logger.error(status) status = status + ":" + ",".join([str(x) for x in artist_ids]) return status def process(self) -> Dict[str, Any]: result = { "non_signed_artists": self._check_exist_data_non_signed_artists(), "artist_ids_type": self._check_id_non_signed_artist(), "artists_dsp_label": self._check_artist_same_label(), "excluded_labels": self._check_excluded_labels(), "excluded_artists": self._check_excluded_artists(), "dead_artists": self._check_dead_artists(), } if all(x == OK_STATUS for x in result.values()): self.logger.info(result) else: self.logger.error(result) return result