import json import math import os from concurrent.futures.thread import ThreadPoolExecutor from datetime import datetime from time import sleep import requests from sme_logger import get_logger from chartmetric.aws_utils import AwsUtils from const import APP_NAME, ENV, aws, chartmetric from endpoints.refreshToken import ChartmetricTokenRefresh from utility import Utilities, measure # - - - - Variables for Performance Logs - - - - # process_type = "Get Chartmetric IDs" last_run = Utilities().get_last_run(process_type) start_time = datetime.today().strftime("%Y-%m-%d %H:%M:%S") today = datetime.today().strftime("%Y-%m-%d") # - - - - Variables for Performance Logs - - - - # logger = get_logger(APP_NAME, os.environ.get("ENVIRONMENT", ENV).lower() != ENV) class FinalData(object): final_data = [] count = 0 class GetChartmetricId(object): def __init__(self): self.spotify_id = "" self.artist_name = "" self.url = "" self.artists = Utilities().fetch_artists_name_from_postgres() self.token = ChartmetricTokenRefresh().refresh_chartmetric_token() self.headers = {"Authorization": f"Bearer {self.token}"} self.payload = "" self.db = AwsUtils().connect_to_database() self.cur = self.db.cursor() self.no_ids_fetched = [] self.commands = [] def attempt_name_search(self, artist): artist_name = artist url = (chartmetric.get("urls", None).get( "CM_CHARTMETRIC_ID_URL_NAME_SEARCH", None).format(artist_name.replace(" ", ""))) try: response = requests.request("GET", url, headers=self.headers) response_dict = json.loads(response.text) logger.debug( "response_dict retrieved contains artist data: %s", response_dict["obj"], ) if "obj" in response_dict: data = response_dict["obj"] if data is not None and len(data) >= 1: qq = ( aws.get("RDS", None).get("queries", None).get( "INSERT_ARTIST_QUERY_USING_NAME", None).format( data["artists"][0]["id"], # Chartmetric ID artist_name.replace("'", "''"), # Artist Name )) logger.debug( f"Invalid data received for Name: {artist_name}") self.commands.append(qq) else: logger.error( f"Invalid data received for Name: {artist_name}") self.no_ids_fetched.append(artist_name) else: logger.error(f"Invalid data received for Name: {artist_name}") self.no_ids_fetched.append(artist_name) except Exception as e: # most generic exception you can catch logger.error( f"Exception while collecting artist data using name: {artist_name} : {e}" ) def get_chartmetric_id_spotify_search(self, artist_list): for artist in artist_list: FinalData.count += 1 if FinalData.count == 900: sleep(60) spotify_id = artist[2] url = (chartmetric.get("urls", None).get( "CM_CHARTMETRIC_ID_URL_SPOTIFY_SEARCH", None).format(spotify_id)) try: response = requests.request("GET", url, headers=self.headers) response_dict = json.loads(response.text) logger.debug( "response_dict retrieved contains artist data: %s", response_dict["obj"], ) if "obj" in response_dict: data = response_dict["obj"] if len(data) >= 1: qq = ( aws.get("RDS", None).get("queries", None).get( "INSERT_ARTIST_QUERY_USING_SPOTIFY", None).format( data[0]["cm_artist"], # Chartmetric ID spotify_id, # Spotify ID )) self.commands.append(qq) else: self.attempt_name_search(artist[3]) logger.error( f"Invalid data received for Spotify ID: {spotify_id}, Attempting name search" ) else: self.attempt_name_search(artist[3]) logger.error( f"Invalid data received for Spotify ID No obj: {spotify_id}, Attempting name " f"search") except Exception as e: # most generic exception you can catch logger.error( f"Exception while collecting artist data using Spotify ID! : {e}" ) @measure def run(self): current_limit = chartmetric.get("RATE_LIMIT", None) runs = math.floor(current_limit / 5) pool = ThreadPoolExecutor(runs) count = 0 for i in range(len(self.artists)): if count > runs: self.db = AwsUtils().connect_to_database() try: for items in self.commands: self.db.autocommit = True self.cur.execute(items) self.commands.pop(self.commands.index(items)) logger.info("Executing Queries") except Exception as e: logger.error(f"Exception: {e}") sleep(20) ChartmetricTokenRefresh().refresh_chartmetric_token() FinalData.final_data = [] count = 0 pool.submit(self.get_chartmetric_id_spotify_search, self.artists[i:i + 1]) self.artists.pop(i) count += 1 pool.shutdown() if __name__ == "__main__": GetChartmetricId().run() end_time = datetime.today().strftime("%Y-%m-%d %H:%M:%S") Utilities().insert_performance_logs(process_type, f"{last_run} + 1", start_time, end_time) time_delta_format = "%Y-%m-%d %H:%M:%S" time_delta = datetime.strptime(end_time, time_delta_format) - datetime.strptime( start_time, time_delta_format) logger.info( f"{process_type}: Start time: {start_time}, End Time: {end_time}, Total Execution Time: {time_delta}" )