import json import os from datetime import date, datetime, timedelta from functools import wraps from time import time import psycopg2 from sme_logger import get_logger from aws_utils import AwsUtils from const import APP_NAME, ENV, aws logger = get_logger(APP_NAME, os.environ.get("ENVIRONMENT", ENV).lower() != ENV) class Utilities(object): @staticmethod def get_database_access(): return AwsUtils().connect_to_database() def get_last_processed_date(self) -> str: """ :return: Date when fan_metrics was last fetched """ db = self.get_database_access() cur = db.cursor() try: cur.execute( aws.get("RDS", None).get("queries", None).get("GET_LAST_UPDATED", None)) last_update_date = cur.fetchone()[0] return last_update_date except Exception as e: logger.error(f"Error getting last processed date: {e}") return str(date.today() - timedelta(days=2)) def get_last_run(self, _process_type) -> str: """ :return: Date when fan_metrics was last fetched """ db = self.get_database_access() cur = db.cursor() db.autocommit = True try: query = str( aws.get("RDS", None).get("queries", None).get( "GET_LATEST_RUN_COUNT", None)).format(date.today(), _process_type) cur.execute(query) last_run_date = cur.fetchone() if last_run_date[0] is None: last_run_date = 0 else: last_run_date = last_run_date[0] return last_run_date except Exception as e: logger.error( f"Error getting last run date from Performance logs: {e}") def insert_performance_logs(self, _process_type, schedule, _start_time, _end_time, parent_id): """ :return: Date when fan_metrics was last fetched """ try: db = self.get_database_access() cur = db.cursor() db.autocommit = True query = str( aws.get("RDS", None).get("queries", None).get("INSERT_EXECUTION_TIME_METRICS", None)).format( _process_type, schedule, _start_time, _end_time, _end_time, _start_time, date.today(), parent_id, ) cur.execute(query) logger.info(f"Performance table updated for {_process_type}") except Exception as e: logger.error(f"Error Inserting performance logs: {e}") def insert_performance_logs_fan_metrics(self, _process_type, schedule, _start_time, parent_id): """ :return: Date when fan_metrics was last fetched """ try: db = self.get_database_access() cur = db.cursor() db.autocommit = True query = str( aws.get("RDS", None).get("queries", None).get( "INSERT_EXECUTION_TIME_FAN_METRICS", None)).format(_process_type, schedule, _start_time, date.today(), parent_id) cur.execute(query) logger.info(f"Performance table updated for {_process_type}") logger.info(f"Performance table updated for {_process_type}") except Exception as e: logger.error(f"Error Inserting performance logs: {e}") def update_performance_logs_fan_metrics(self, _end_time, _start_time, stats, _process_type, schedule): """ :return: Date when fan_metrics was last fetched """ try: db = self.get_database_access() cur = db.cursor() db.autocommit = True query = str( aws.get("RDS", None).get("queries", None).get( "UPDATE_EXECUTION_TIME_FAN_METRICS", None)).format( _end_time, _end_time, _start_time, stats, _process_type, schedule, _start_time, ) cur.execute(query) logger.info(f"Performance table updated for {_process_type}") except Exception as e: logger.error(f"Error Inserting performance logs: {e}") def get_artists_from_postgres(self) -> list: """ :return: List of artists with their chartmetric_ids """ db = self.get_database_access() cur = db.cursor() try: cur.execute( aws.get("RDS", None).get("queries", None).get("GET_ARTIST_CHARTMETRIC_IDS", None)) return cur.fetchall() except psycopg2.OperationalError as e: logger.error(f"Error getting artists from postgres: {e}") def fetch_artists_name_from_postgres(self) -> list: """ :return: List of artists names, used to fetch chartmetric id's """ db = self.get_database_access() cur = db.cursor() try: cur.execute( aws.get("RDS", None).get("queries", None).get("FETCH_ARTIST_CHARTMETRIC_ID", None)) return cur.fetchall() except psycopg2.OperationalError as e: logger.error(f"Error getting artist names from postgres: {e}") def get_artists_from_reproc(self) -> list: """ :return: list of artist from chartmetric status table to retry fetching the fan_metrics for them """ db = self.get_database_access() cur = db.cursor() try: cur.execute( aws.get("RDS", None).get("queries", None).get( "GET_ARTIST_CHARTMETRIC_IDS_REPROC", None)) return cur.fetchall() except psycopg2.OperationalError as e: logger.error(f"Error getting artists from chartmetric_status: {e}") # - - - - - - - - Put artists to Postgres DB - - - - - - - - # def put_logs_to_postgres(self, query: str, artist_gras_id_list: list): """ :param query: Query which needs to be executed to post logs to status logs :param artist_gras_id_list: list of artists along with sources which need a retry attempt :return: None """ # ----------- POSTGRESQL SECTION STARTS------------------# db = self.get_database_access() db.autocommit = True cur = db.cursor() try: cur.execute( aws.get("RDS", None).get("queries", None).get("LAST_DATE_QUERY", None)) except psycopg2.OperationalError as e: logger.error(f"Operational Error: {e}") last_date = cur.fetchone() date_time = datetime.date(datetime.now()) if last_date[0] is not None and str(last_date[0]) != str(date_time): try: cur.execute( aws.get("RDS", None).get("queries", None).get("INSERT_REPROC_EXEMPT", None)) cur.execute( aws.get("RDS", None).get("queries", None).get("TRUNCATE_STATUS_TABLE", None)) logger.info( f"chartmetric_status_exempt has been updated and chartmetric_status has been truncated" ) except psycopg2.OperationalError as e: logger.error(f"Error updating tables: {e}") artist_gras_id_list = [int(i) for i in artist_gras_id_list] if artist_gras_id_list: query = ( query + "\n" + aws.get("RDS", None).get("queries", None).get( "COUNT_MANAGE_QUERY", None).format(artist_gras_id_list)) query = query.replace("[", "(").replace("]", ")") try: cur.execute(query) except psycopg2.OperationalError as e: logger.error(f"Operational Error: {e}") def logs_to_postgres(self, logs: dict, retry_attempt: bool): artist_gras_id_list = [] commands = "" columns = {} for id, content in logs.items(): columns["gras_id"] = int(id) artist_gras_id_list.append(id) for key, val in content.items(): if key == "invalid_request": columns["chartmetric_id"] = val["chartmetric_id"] continue for k, v in val.items(): if k == "chartmetric_id": columns[k] = int(v) elif k == "request": columns[key + "_req"] = str(v) elif k == "response": columns[key + "_res"] = str(v) elif k == "update_date": columns["update_date"] = v elif k == "valid_data": columns[key + "_val"] = str(v) update_dict = ",".join([ kk + "=" + "'" + vv + "'" if type(vv) == str else kk + "=" + str(vv) for kk, vv in columns.items() ]) if not retry_attempt: query = (aws.get("RDS", None).get("queries", None).get( "INSERT_ARTIST_LOGS_QUERY", None).format( json.dumps(tuple(columns.keys())), list(columns.values()), update_dict, )) commands = commands + query + "\n" if retry_attempt: query = (aws.get("RDS", None).get("queries", None).get( "INSERT_ARTIST_LOGS_QUERY", None).format( json.dumps(tuple(columns.keys())), list(columns.values()), update_dict, )) commands = commands + query + "\n" self.put_logs_to_postgres(commands, artist_gras_id_list) logger.info("Status Logs sent to database") @staticmethod def set_google_credentials(secret): # - - - - - - Set the google credentials as Environment Variable - - - - - - # try: secret = AwsUtils().get_secret(required_secret=secret) with open("google_credentials.json", "w") as file: file.write(secret) # set the environment variable os.environ[ "GOOGLE_APPLICATION_CREDENTIALS"] = "google_credentials.json" logger.debug("Google Credentials set successfully") except Exception as e: logger.error("Google Credentials exception: {0}".format(e)) def execution_start_time(): # dd/mm/YY H:M:S now = datetime.now() return now.strftime("%d/%m/%Y %H:%M:%S") def measure(func): @wraps(func) def _time_it(*args, **kwargs): start = int(round(time() * 1000)) try: return func(*args, **kwargs) finally: end_ = int(round(time() * 1000)) - start logger.debug( f"Total execution time: {end_ if end_ > 0 else 0} mili-seconds" ) return _time_it