"""Dimension Refresh CLI. ======================== Cli script for executing a dim refresh. """ from argparse import ArgumentParser from datetime import datetime from datetime import timedelta import json import logging import sys import time from garcon import activity from garcon import decider from dim_refresh_etl.util import import_utils # TODO(jpenner): move to garcon DeciderWorker repo def run_decider(decider): """Run decider daemon. Args decider (DeciderWorker): DeciderWorker to run """ while True: decider.run() def start(flow, context=None): """Start a flow. Args: flow (DimWorkFlow): Garcon Workflow to execute context (dict): the context to pass to the flow. Should have dim_type set as the dimension being refreshed Return: WorkflowExecution: executing the workflow returns information about the execution. """ start_to_close_timeout = 7200 if hasattr(flow, 'timeout'): start_to_close_timeout = flow.timeout workflow_id = flow.workflow_id(context) print('Starting workflow {} on domain {} with id {}'.format( flow.name, flow.domain, workflow_id)) execution = flow.client.start_workflow_execution( domain=flow.domain, workflowId=workflow_id, workflowType={'name': flow.name, 'version': flow.version}, taskList={'name': flow.name}, executionStartToCloseTimeout=str(start_to_close_timeout), input=json.dumps(context or {})) if context.get('wait_until_complete'): print('Polling until Workflow is complete') run_id = execution['runId'] max_time = datetime.now() + timedelta(minutes=60) while datetime.now() < max_time: current_exec = flow.client.describe_workflow_execution( domain=flow.domain, execution={'workflowId': workflow_id, 'runId': run_id}) if current_exec['executionInfo'].get( 'executionStatus') != 'CLOSED': time.sleep(10) else: if current_exec['executionInfo'].get( 'closeStatus') == 'COMPLETED': return else: raise Exception( 'Workflow with runId {runId} in domain {domain} ' 'failed'.format( runId=run_id, domain=flow.domain)) else: return def main(args=None): """Arg parser method dim_refresh cli.""" parser = ArgumentParser( description='Analytics Worker command line.') parser.add_argument( '-i', '--info', help='set logging to info', action='store_true') parser.add_argument( '-d', '--debug', help='set logging to debug', action='store_true') parser.add_argument( '-c', '--context', dest='context', help='flow context [json string]', default='{}') parser.add_argument( '-cf', '--context-file', help='flow context [json file]') parser.add_argument( choices=['refresh', 'geocoding', 'dynamo_sync'], help='name of the flow', dest='flow') cmd_subparsers = parser.add_subparsers( help='garcon command', dest='cmd') # subparser for running activity locally local_subparsers = cmd_subparsers.add_parser( 'local', help='run flow locally') local_subparsers.add_argument('dimension', help='dimension to refresh') local_subparsers.add_argument('activity', help='activity to run') # subparser for executing flow exec_subparser = cmd_subparsers.add_parser('exec', help='start flow') exec_subparser.add_argument( 'dimension', help='dimension to run against.') exec_subparser.add_argument( '--wait-until-complete', help='not return until workflow is complete', action='store_true') # subparser for running an activity worker worker_subparser = cmd_subparsers.add_parser( 'worker', help='run activity worker') worker_subparser.add_argument( '-a', '--activities', dest='activities', default=None, help='define the name of the activity (activity, decider).') # subparser for running decider cmd_subparsers.add_parser('decider', help='run decider') # parse cl args (allows easy unit testing) args = parser.parse_args(args) if args else parser.parse_args() if len(sys.argv) <= 1: parser.print_help() exit() # respect most verbose level of logging set if args.debug: logging.basicConfig(level=logging.DEBUG) elif args.info: logging.basicConfig(level=logging.INFO) context_json = json.loads(args.context) # context file wins if both context str & file passed if args.context_file: with open(args.context_file) as context_file: context_json = json.load(context_file) flow_class = import_utils.import_workflow_class(args.flow) flow = flow_class() if args.cmd == 'exec': context_json['dim_type'] = args.dimension if args.wait_until_complete: context_json['wait_until_complete'] = True status = start(flow, context_json) sys.exit(status) # 0 or 1 elif args.cmd == 'decider': decider_worker = decider.DeciderWorker(flow) run_decider(decider_worker) elif args.cmd == 'worker': activity_worker = activity.ActivityWorker(flow, args.activities) activity_worker.run() else: sys.exit('Bad Command') if __name__ == '__main__': main()