"""Unified connector to loguru-based logger with various configurations.""" import os import sys from datetime import datetime from uuid import uuid4 from loguru import logger from owslogger.logger import DSNHandler from integration_scripts.config import MASTER_LOG, JSON_LOG, CSV_LOG, \ OTHER_LOG, DEBUG_LOG, LOG_DIR, LOGGER_NAME, LOGGER_DSN, LOGGER_LEVEL,\ ENVIRONMENT, CONSOLE_LOG_LEVEL, APP_VERSION from integration_scripts.file_utils import create_path_with_todays_date class LoguruDSNHandler(DSNHandler): """Loguru OWSLogging DNS Handler. Custom DSN handler that sends a JSON payload complying with OWS2 standard. """ def emit(self, record): """Emit a record. From the documentation: do whatever it takes to actually log the specified logging record. Here: we send it to the provider. The payload matches Orchard format. Args: record (LogRecord): the record to log. """ record.correlation_id = record.extra.get('correlation_id') record.session_id = record.extra.get('session_id') super().emit(record) default_format = '{time:YYYY-MM-DD at h:mm:ss A zz} | {' \ 'level} | {message}' if MASTER_LOG and '{timestamp}' in MASTER_LOG: MASTER_LOG = MASTER_LOG.format(timestamp=datetime.today().strftime( '%Y-%m-%d_%H_%M_%S')) if CSV_LOG and '{timestamp}' in CSV_LOG: CSV_LOG = CSV_LOG.format(timestamp=datetime.today().strftime( '%Y-%m-%d_%H_%M_%S')) if JSON_LOG and '{timestamp}' in JSON_LOG: JSON_LOG = JSON_LOG.format(timestamp=datetime.today().strftime( '%Y-%m-%d_%H_%M_%S')) if OTHER_LOG and '{timestamp}' in OTHER_LOG: OTHER_LOG = OTHER_LOG.format(timestamp=datetime.today().strftime( '%Y-%m-%d_%H_%M_%S')) def default_filter(record): return all( v not in record["extra"] for v in [ 'json_only', 'file_only', 'csv_only', 'other_only', 'multi' ]) and record["level"].no >= logger.level(CONSOLE_LOG_LEVEL).no \ and record["level"].no != logger.level("ERROR").no def default_error_filter(record): return all( v not in record["extra"] for v in [ 'json_only', 'file_only', 'csv_only', 'other_only', 'multi' ]) and record["level"].no >= logger.level("ERROR").no def default_debug_filter(record): return all( v not in record["extra"] for v in [ 'json_only', 'file_only', 'csv_only', 'other_only', 'multi' ]) and record["level"].no == logger.level("DEBUG").no def default_multi_filter(record): return all( v not in record["extra"] for v in [ 'json_only', 'file_only', 'csv_only', 'other_only' ]) and record["level"].no >= logger.level(CONSOLE_LOG_LEVEL).no \ and record["level"].no != logger.level("ERROR").no \ and 'multi' in record['extra'] def default_multi_error_filter(record): return all( v not in record["extra"] for v in [ 'json_only', 'file_only', 'csv_only', 'other_only' ]) and record["level"].no >= logger.level("ERROR").no \ and 'multi' in record['extra'] def default_multi_debug_filter(record): return all( v not in record["extra"] for v in [ 'json_only', 'file_only', 'csv_only', 'other_only' ]) and record["level"].no == logger.level("DEBUG").no \ and 'multi' in record['extra'] # Ensure handlers are only added once. try: # Remove and replace default logger logger.remove(0) except ValueError: pass else: # Default Main Logger logger.add(sys.stderr, format=default_format, filter=default_filter, colorize=True, level=CONSOLE_LOG_LEVEL) # Default Error Logger logger.add(sys.stderr, filter=default_error_filter, colorize=True, level='ERROR') # Default Main Enqueued Logger logger.add(sys.stderr, format=default_format, filter=default_multi_filter, colorize=True, enqueue=True, level=CONSOLE_LOG_LEVEL) # Default Error Enqueued Logger logger.add(sys.stderr, filter=default_multi_error_filter, colorize=True, enqueue=True, level='ERROR') # Debug level loggers if DEBUG_LOG: logger.add(sys.stderr, filter=default_debug_filter, level='DEBUG') logger.add(sys.stderr, filter=default_multi_debug_filter, enqueue=True, level='DEBUG') # PropogateHandler for ows-logger if LOGGER_DSN: # bind context vars for datadog correlation_id = uuid4() context = { 'correlation_id': correlation_id } logger.add( LoguruDSNHandler( LOGGER_DSN, ENVIRONMENT, LOGGER_NAME, APP_VERSION), level=LOGGER_LEVEL, filter=default_filter) logger.add( LoguruDSNHandler( LOGGER_DSN, ENVIRONMENT, LOGGER_NAME, '1.0'), level=LOGGER_LEVEL, filter=default_multi_filter) logger.add( LoguruDSNHandler( LOGGER_DSN, ENVIRONMENT, LOGGER_NAME, APP_VERSION), level='ERROR', filter=default_error_filter) logger.add( LoguruDSNHandler(LOGGER_DSN, ENVIRONMENT, LOGGER_NAME, '1.0'), level='ERROR', filter=default_multi_error_filter) if DEBUG_LOG: logger.add( LoguruDSNHandler(LOGGER_DSN, ENVIRONMENT, LOGGER_NAME, '1.0'), level='DEBUG', filter=default_debug_filter) logger.add( LoguruDSNHandler(LOGGER_DSN, ENVIRONMENT, LOGGER_NAME, '1.0'), level='DEBUG', filter=default_multi_debug_filter) # Bind Correlation_ID to logger logger = logger.bind(**context) logger.info(f'Correlation ID: {correlation_id}') # FILE LOGS --------------------------------------------------------------- if MASTER_LOG or CSV_LOG or JSON_LOG or OTHER_LOG: os.makedirs(create_path_with_todays_date( LOG_DIR, subdir=True, timestamp=True), exist_ok=True) # -- Text Log ------------------------------------------------------------- if MASTER_LOG: logger.info('Writing console log to {}', MASTER_LOG) logger.add(os.path.join(LOG_DIR, MASTER_LOG), rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['file', 'file_only']) and 'multi' not in record[ 'extra']) # Enqueued logger.add(os.path.join(LOG_DIR, MASTER_LOG), rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['file', 'file_only']) and 'multi' in record['extra'], enqueue=True) # -- JSON Log ------------------------------------------------------------- if JSON_LOG: logger.add(os.path.join(LOG_DIR, JSON_LOG), serialize=True, rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['json', 'json_only']) and 'multi' not in record[ 'extra']) # Enqueued logger.add(os.path.join(LOG_DIR, JSON_LOG), serialize=True, rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['json', 'json_only']) and 'multi' in record['extra'], enqueue=True) # -- CSV Log -------------------------------------------------------------- if CSV_LOG: logger.add(os.path.join(LOG_DIR, CSV_LOG), format='{message}', rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['csv', 'csv_only']) and 'multi' not in record['extra']) # Enqueued logger.add(os.path.join(LOG_DIR, CSV_LOG), format='{message}', rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['csv', 'csv_only']) and 'multi' in record['extra'], enqueue=True) # Secondary Text Log if OTHER_LOG: logger.add(os.path.join(LOG_DIR, OTHER_LOG), format='{message}', rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['other', 'other_only']) and 'multi' not in record[ 'extra']) # Enqueued logger.add(os.path.join(LOG_DIR, OTHER_LOG), format='{message}', rotation="5 MB", retention='30 days', filter=lambda record: any( v in record["extra"] for v in ['other', 'other_only']) and 'multi' in record['extra'], enqueue=True) # # CSV_LOG EXAMPLE # logger.bind(csv=True).debug('csv_field_1, csv_field_2, etc.') # logger.bind(csv=True).debug('val_1_1, val_1_2, etc.') # logger.bind(csv=True).debug('val_2_1, val_2_2, etc.') # # OTHER_LOG EXAMPLE # logger.debug('Message in only MASTER_LOG') # logger.bind(other=True).debug('Message in both MASTER_LOG & OTHER_LOG') # Multiprocessing / Threading # logger.bind(multi=True).debug('Enqueued message for concurrency.' # welcome message logger.debug("Loguru logging enabled.")