"""Accounting ETL status log specific functions. This has functions for all things related to status logging in accounting_revenue_etl_log table. This is separate from general logger. """ from datetime import datetime from flows import datastore from flows import g from flows.sales_data import status def create(correlation_id, accounting_period_id, upcs): """Create a new ETL log entry. Args: correlation_id (str): Correlation ID for ETL process. accounting_period_id (int): account period to process. upcs (DatabaseParam): upc data. """ sql = """ INSERT accounting_revenue_etl_log (correlation_id, accounting_period_id, etl_start, etl_status, upcs) VALUES ( %(correlation_id)s, %(accounting_period_id)s, %(etl_start)s, %(etl_status)s, %(upcs)s )""" params = { 'correlation_id': correlation_id, 'accounting_period_id': accounting_period_id, 'upcs': upcs.data_json, 'etl_start': datetime.now(), 'etl_status': status.STARTED} datastore.execute(sql, params) def add_run_id(correlation_id, workflow_run_id): """Update ETL log with run ID from AWS SWF. Args: correlation_id (str): Correlation ID for ETL process. workflow_run_id (str): SWF execution run id. """ sql = """ UPDATE accounting_revenue_etl_log SET workflow_run_id = %(workflow_run_id)s WHERE correlation_id = %(correlation_id)s""" params = { 'correlation_id': correlation_id, 'workflow_run_id': workflow_run_id} datastore.execute(sql, params) def update_status(correlation_id, new_status, end_of_etl=False): """Update ETL log status to new_status. Args: correlation_id (str): Correlation ID for ETL process. new_status (str): new status for the etl log. end_of_etl (bool): Indicate if the etl has completed its execution. """ etl_end = None if end_of_etl: etl_end = datetime.now() sql = """ UPDATE accounting_revenue_etl_log SET etl_status = %(status)s, etl_end = %(etl_end)s WHERE correlation_id = %(correlation_id)s""" params = { 'correlation_id': correlation_id.split('.')[0], 'status': new_status, 'etl_end': etl_end} g.log.info( 'Updating sales_data ETL with correlation ID: "{correlation_id}" ' 'to status: "{status}"'.format(**params)) datastore.execute(sql, params) def accounting_period_update(correlation_id, accounting_period_id): """Update ETL log with accounting_period_id from lookup. Args: correlation_id (str): Correlation ID for ETL process. accounting_period_id (int): Accounting Period ID for ETL process. """ sql = """ UPDATE accounting_revenue_etl_log SET accounting_period_id = %(accounting_period_id)s WHERE correlation_id = %(correlation_id)s""" params = { 'correlation_id': correlation_id.split('.')[0], 'accounting_period_id': accounting_period_id} g.log.info( 'Updating sales_data ETL with correlation ID: "{correlation_id}" ' 'with accounting period: "{accounting_period_id}"'.format(**params)) datastore.execute(sql, params)