#! /usr/bin/env python3 from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from contextlib import closing import snowflake.connector import psycopg2 import smtplib import argparse import sys import json import datetime import logging with open('config.json', 'r') as f: config = json.load(f) local_db_name = config['failure_counter']["database"] local_db_user = config['failure_counter']["user"] local_db_password = config['failure_counter']["password"] local_db_host = config['failure_counter']["host"] local_db_port = config['failure_counter']["port"] failure_count = "select failed_count from failed_count where count_id='yttch';" def create_arg_parser(): parser = argparse.ArgumentParser() parser.add_argument('-m', '--mail', nargs='+', action='store', dest='recipients', type=str) parser.add_argument('-p', '--print', action="store_const", const=True) return parser def send_mail(recipients: str, message: str, status: str) -> None: # Create a multipart message msg = MIMEMultipart() body_part = MIMEText(message, 'plain') msg['Subject'] = f"[{status}][PROD] YouTube Trending Charts" msg['From'] = "support.sme@dataart.com" msg['To'] = recipients # Add body to email msg.attach(body_part) # Create SMTP object server = smtplib.SMTP('relay1.dataart.com', 25) server.sendmail(msg['From'], msg['To'].split(','), msg.as_string()) server.quit() def postgres_fetch_data(sql_query) -> list: result_list = [] with closing(psycopg2.connect(database=local_db_name, user=local_db_user, password=local_db_password, host=local_db_host, port=local_db_port)) as connection: with connection.cursor() as cursor: cursor.execute(sql_query) columns = cursor.description rows = cursor.fetchall() for row in rows: tmp = {} for i in range(len(columns)): tmp[columns[i][0]] = row[i] result_list.append(tmp) return result_list ################# run the all steps ###################### def run_steps() -> dict: message_dict = {'name':"[PROD] YouTube Trending Charts",'status':"",'date':"", 'info':"", 'details':"", 'full_log':""} # set date according to sql searching date, currently it's `today` message_dict["date"] = (datetime.datetime.now() - datetime.timedelta(days=0)).strftime("%Y-%m-%d") count_yttch = int(postgres_fetch_data(failure_count)[0]['failed_count']) # print(f"Failed count = {count_yttch}") monitoring_dict = {} try: with open('/tmp/Youtube_Trending_charts_hourly_log.json', 'r') as f: monitoring_dict = json.load(f) except Exception as e: logging.error(e) # print(f"{monitoring_dict['details']}") message_dict['full_log'] = monitoring_dict['full_log'] if count_yttch >= 10: if monitoring_dict['step'] == "1": # Data is partially synced with Chartmetric message_dict['status'] = "FAILED" message_dict['info'] = "Data is partially synced with Chartmetric" message_dict['details'] = monitoring_dict['details'] elif monitoring_dict['step'] == "2": # Data is synced with Chartmetric and partially available in API message_dict['status'] = "FAILED" message_dict['info'] = "Data is synced with Chartmetric and partially available in API" message_dict['details'] = monitoring_dict['details'] else: message_dict['status'] = "OK" message_dict['info'] = "Data is synced with Chartmetric and available in API" return message_dict if __name__ == '__main__': # # check command line parameters and print or email result parser = create_arg_parser() arg_list = parser.parse_args() if len(sys.argv[1:]) > 0: result = run_steps() with open('/tmp/Youtube_Trending_charts_daily_log.json', 'w') as json_file: json.dump(result, json_file, indent=4) if arg_list.recipients: recipients = arg_list.recipients[0] send_mail(recipients=recipients, message=json.dumps(result, indent=4), status=result['status']) if arg_list.print: print(json.dumps(result, indent=4)) else: parser.print_help()