import json import os import sys from datetime import datetime import boto3 import psycopg2 from sme_logger import get_logger from aws_utils import AwsUtils from const import APP_NAME, ENV, aws, chartmetric from utility import Utilities, measure # - - - - Variables for Performance Logs - - - - # process_type = "s3 to postgres" last_run = Utilities().get_last_run(process_type) start_time = datetime.today().strftime("%Y-%m-%d %H:%M:%S") today = datetime.today().strftime("%Y-%m-%d") # - - - - Variables for Performance Logs - - - - # logger = get_logger(APP_NAME, os.environ.get("ENVIRONMENT", ENV).lower() != ENV) class SendFromS3ToPostgres(object): def __init__(self): self.s3 = boto3.resource("s3") self.today = datetime.utcnow().date() self.db = AwsUtils().connect_to_database() self.cur = self.db.cursor() self.bucket = self.s3.Bucket( aws.get("s3", None).get("BUCKET_QUARANTINE", None)) @measure def send_chartmetric_ids_to_postgres(self): commands = "" # Read all the files from S3 Quarantine try: for obj in self.bucket.objects.filter( Prefix= f"chartmetric/{chartmetric.get('CHARTMETRIC_VERSION', None)}/" ): key = obj.key if key.startswith( f"{aws.get('s3', None).get('BUCKET_PATH', None)}/"): logger.info(f"Searched for ID's in {key}") logger.debug( f"\n # - - - - - - - - Searched for ID's in {key} - - - - - - - - #" ) body = obj.get()["Body"].read() if body: try: json_content = json.loads(body) if type(json_content) == list: for row in json_content: qq = (aws.get("RDS", None).get( "queries", None ).get("INSERT_ARTIST_QUERY_USING_SPOTIFY", None).format( row["participant_id"], row["artist_name"], row["chartmetric_id"], row["spotify_id"], row["chartmetric_id"], row["spotify_id"], )) commands = commands + qq + "\n" logger.info( f"Chartmetric ID dumped into postgres from s3 Bucket. {self.bucket}" ) except Exception as e: logger.error( f"Exception while reading data from S3 to Postgres:{e}" ) if commands: try: self.cur.execute(commands) logger.info( "Query Execution to send chartmetric Id to postgres completed" ) except psycopg2.OperationalError as e: logger.error( "Query Execution to send chartmetric Id to postgres Error: {0}" .format(e)) sys.exit(0) except Exception as e: logger.warning( f"Exception in reading files from s3 bucket {self.bucket} : {e}", exc_info=True, ) if __name__ == "__main__": SendFromS3ToPostgres().send_chartmetric_ids_to_postgres() end_time = datetime.today().strftime("%Y-%m-%d %H:%M:%S") Utilities().insert_performance_logs(process_type, f"{last_run} + 1", start_time, end_time) time_delta_format = "%Y-%m-%d %H:%M:%S" time_delta = datetime.strptime(end_time, time_delta_format) - datetime.strptime( start_time, time_delta_format) logger.info( f"{process_type}: Start time: {start_time}, End Time: {end_time}, Total Execution Time: {time_delta}" )