"""Utility script to re-run failed custom export executions.""" import argparse import datetime import logging import sys import boto3 from processing_accounting.flows.custom_export import tasks from processing_accounting.util import dynamodb from processing_accounting.util import swf DEFAULT_DOMAIN = 'prod_swf_processing_accounting' DEFAULT_DYNAMODB_TABLE = 'prod_accounting_statement_export' DATE_FORMAT = '%m/%d/%Y' logger = logging.getLogger(__name__) logger.setLevel(logging.INFO) regular_formatter = '%(asctime)s: %(message)s' stream_handler_formatter = logging.Formatter(regular_formatter) stream_handler = logging.StreamHandler(stream=sys.stdout) stream_handler.setFormatter(stream_handler_formatter) stream_handler.setLevel(logging.INFO) logger.addHandler(stream_handler) def get_session(profile_name='default'): """Get AWS session using profile name credentials. Args: profile_name (str): configured AWS profile name Returns: boto3.session.Session: AWS session """ session = boto3.session.Session(profile_name=profile_name) return session def get_dynamodb_key_from_input(execution_input): """Build DynamoDB key from execution input. Args: execution_input (dict): workflow execution input Returns: dict: DynamoDB key """ user_id_type = tasks._get_user_id_type( user_id=execution_input['user_id'], user_type=execution_input['user_type']) user_params = tasks._get_user_params( period_ids=execution_input['period_ids'], transaction_types=execution_input['transaction_types'], locale=execution_input['locale'], file_format=execution_input['file_format'] ) return {'user_id_type': user_id_type, 'user_params': user_params} def get_date(date_str): """Convert str date to datetime.datetime object. Args: date_str (str): %d/%m/%Y formatted date. E.g. 03/08/2018 Returns: datetime.datetime: datetime object """ msg = ( 'Invalid date input, please make sure your date is {} ' 'formatted. E.g. 01/15/2018'.format(DATE_FORMAT)) try: result = datetime.datetime.strptime(date_str, DATE_FORMAT) except ValueError: raise argparse.ArgumentTypeError(msg) return result def process_items(items, domain_name, table_name, session, re_run=False): """Check and re-run failed executions. Args: items (list): failed executions domain_name (str): SWF execution domain name table_name (str): DynamoDB table name session (boto3.session.Session): AWS session re_run (bool): if True - will re-run the failed executions Returns: generator: list of failed executions with input """ for item in items: execution = item['execution'] execution_history = swf.get_execution_history( domain_name, execution, session=session) execution_input = swf.get_execution_input(execution_history) key = get_dynamodb_key_from_input(execution_input) status = dynamodb.get_status_using_session( key=key, table_name=table_name, session=session) if status not in ('PENDING', 'GENERATING', ''): continue failed_execution = { 'execution_data': item, 'input': execution_input, } if not re_run: yield failed_execution continue resp = swf.re_run_execution( execution=item, execution_input=execution_input, domain_name=domain_name, session=session) failed_execution['rerun_response'] = resp yield failed_execution def main( profile_name, domain_name, table_name, latest_date, time_delta, re_run): """Script main logic. Args: profile_name (str): AWS shared credentials profile name domain_name (str): SWF execution domain name table_name (str): DynamoDB table name latest_date (datetime.datetime): Latest date (latest - time delta) time_delta (int): Time delta in days re_run (bool): Flag to re-run the failed executions """ logger.info('Initiating AWS Session using %s profile', profile_name) session = get_session(profile_name=profile_name) msg = ( 'Retrieving failed executions. Using:\n' ' domain_name: %(domain_name)s\n' ' table_name: %(table_name)s\n' ' latest_date: %(latest_date)s\n' ' time_delta: %(time_delta)d') logger.info(msg, { 'domain_name': domain_name, 'table_name': table_name, 'latest_date': latest_date.isoformat(), 'time_delta': time_delta}) failed_executions = swf.get_latest_failed_executions( domain=domain_name, latest_date=latest_date, session=session, days_range=time_delta) logger.info( 'Found %s failed execution(s).', len(failed_executions)) if not failed_executions: return failed_executions = process_items( failed_executions, domain_name=domain_name, table_name=table_name, session=session, re_run=re_run) logger.info('{:=^79}'.format(' Verified execution(s) details ')) for execution in failed_executions: for key, value in execution.items(): logger.info('{:-^79}'.format(' ' + key + ' ')) logger.info(value) logger.info('-' * 79) def entry_point(): """Script entry point.""" parser = argparse.ArgumentParser( description=( 'Re-run failed custom export flow executions. ' 'The script checks the SWF execution history and ' 'the execution status in DynamoDB before re-running.')) parser.add_argument( '-p', '--profile-name', help='Shared AWS credentials profile name.', default='default') parser.add_argument( '-t', '--table-name', default=DEFAULT_DYNAMODB_TABLE, help='DynamoDB table name. Defaults to PROD table.') parser.add_argument( '-d', '--domain-name', default=DEFAULT_DOMAIN, help='SWF execution domain. Defaults to PROD domain.') now = datetime.datetime.now() default_latest_date = now + datetime.timedelta(days=1) parser.add_argument( '--latest-date', type=get_date, default=default_latest_date.strftime(DATE_FORMAT), help=( '%d/%m/%Y (21/12/2018) formatted latest date to check. ' 'Defaults to now/tomorrow.')) parser.add_argument( '--time-delta', default=1, type=int, help='Time delta, days. How far to go back in time.\n' 'Calculated as now - time-delta. ' 'E.g. Now - 1 day = 1 day of execution history etc.') parser.add_argument( '-r', '--re-run', action='store_true', help='Re-run failed executions.') args = parser.parse_args() profile_name = args.profile_name table_name = args.table_name domain_name = args.domain_name latest_date = args.latest_date time_delta = args.time_delta re_run = args.re_run main( profile_name=profile_name, domain_name=domain_name, table_name=table_name, latest_date=latest_date, time_delta=time_delta, re_run=re_run, ) if __name__ == '__main__': entry_point()