""" Projections ETL log specific functions. This has convenience functions for all things related to the logs in the projections_etl_log table. It is separate from a general logger. """ from datetime import datetime from flows import datastore from flows import g from flows.projections import status def create(correlation_id, workflow_run_id, upc, file_date, log_table): """Create a new ETL log entry. Args: correlation_id (str): Correlation ID for ETL process. workflow_run_id (str): Workflow run id generated by AWS SWF service. upc (int): UPC value. file_date (date): file date from file name. log_table (str): Table name for ETL logs. """ sql = """ INSERT {log_table} ( correlation_id, workflow_run_id, upc, file_date, etl_start, etl_status) VALUES ( %(correlation_id)s, %(workflow_run_id)s, %(upc)s, %(file_date)s, %(etl_start)s, %(etl_status)s) ON DUPLICATE KEY UPDATE correlation_id = %(correlation_id)s, workflow_run_id = %(workflow_run_id)s, upc = %(upc)s, etl_start = %(etl_start)s, etl_status = %(etl_status)s;""".format( log_table=log_table) params = { 'correlation_id': correlation_id, 'workflow_run_id': workflow_run_id, 'upc': upc, 'file_date': file_date, 'etl_start': datetime.now(), 'etl_status': status.STARTED} datastore.execute(sql, params) def update_status(correlation_id, status, log_table): """Update ETL log status. Args: correlation_id (str): Correlation ID for ETL process. status (str): new status for the etl log. log_table (str): Table name for ETL logs. """ sql = """ UPDATE {log_table} SET etl_status = %(status)s WHERE correlation_id = %(correlation_id)s;""".format( log_table=log_table) params = { 'correlation_id': correlation_id.split('.')[0], 'status': status} g.log.info( 'Updating ETL with correlation ID: "{correlation_id}" ' 'to status: "{status}"'.format(**params)) datastore.execute(sql, params)