"""CLI functions to run decider, worker and exec processes.""" import argparse from datetime import datetime from datetime import timedelta import importlib import json import logging import time import boto3 from garcon import activity from garcon import decider from feed_ingestion import flows from feed_ingestion.conf.config import BOTO3_CONFIG from feed_ingestion.util import sentry_util from feed_ingestion.util.aws import swf as swf_util from feed_ingestion.util.aws.swf import generate_execution_console_url def execute_flow(flow, context, output_file=None, **kwargs): """Launch the workflow execution. Args: flow (module): Garcon flow module. context (str): Initial context parsed from JSON. output_file: Optional path to file to store all args as well as workflow_id and run_id. kwargs: Extensible call API. """ if swf_util.has_nonconcurrent_workflows_running(flow): print( 'Its not allowed to run concurrently with some of the ' 'currently active flows. Exiting') return workflow_id = flow.workflow_id(json.loads(context)) start_kwargs = dict( domain=flow.domain, workflowId=workflow_id, workflowType={ 'name': flow.name, 'version': flow.version, }, taskList={ 'name': flow.name, }, input=context, childPolicy='TERMINATE', ) if hasattr(flow, 'timeout'): start_kwargs.update(executionStartToCloseTimeout=str(flow.timeout)) if hasattr(flow, 'task_timeout'): start_kwargs.update(taskStartToCloseTimeout=str(flow.task_timeout)) swf = boto3.client('swf', config=BOTO3_CONFIG) try: start_response = swf.start_workflow_execution(**start_kwargs) except swf.exceptions.WorkflowExecutionAlreadyStartedFault: print('Workflow execution already started') return run_id = start_response['runId'] print('Created SWF execution:') print(generate_execution_console_url( swf_domain=flow.domain, workflow_id=workflow_id, run_id=run_id, ), flush=True) if output_file: with open(output_file, 'w') as out_file: start_kwargs.update(run_id=run_id) json.dump(start_kwargs, out_file) if json.loads(context).get('wait_until_complete') != 'True': return print('Polling until Workflow is complete', flush=True) max_time = datetime.now() + timedelta(seconds=flow.timeout) while datetime.now() < max_time: current_exec = swf.describe_workflow_execution( domain=flow.domain, execution=dict( workflowId=workflow_id, runId=run_id, ), ) if current_exec['executionInfo'].get( 'executionStatus') != 'CLOSED': time.sleep(10) else: if current_exec['executionInfo'].get( 'closeStatus') == 'COMPLETED': return raise Exception( 'Workflow with runId {runId} in domain {domain} ' 'failed'.format( runId=run_id, domain=flow.domain)) def run_decider(flow, **kwargs): """Launch the SWF decider process. Args: flow (module): Garcon flow module. kwargs (dict): Extensible call API. """ worker = decider.DeciderWorker(flow) while True: worker.run() time.sleep(1) def run_activity_worker(flow, **kwargs): """Launch the activity worker process. Args: flow (module): Garcon flow module. kwargs (dict): Extensible call API. """ worker = activity.ActivityWorker(flow) worker.run() _COMMANDS = {'exec': execute_flow, 'decider': run_decider, 'worker': run_activity_worker} def garcon(*args): """Entry point for the Garcon command line integration. Args: args (tuple): Args for the flow. """ parser = argparse.ArgumentParser(description='Garcon command line util') parser.add_argument('cmd', choices=_COMMANDS.keys(), help='garcon command') parser.add_argument('flow', choices=flows.__all__, help='name of the flow') parser.add_argument('-c', '--context', help='initial context [json]') parser.add_argument('-l', '--log_level', help='logging level', default='error', choices=['critical', 'error', 'warning', 'info', 'debug']) parser.add_argument('-o', '--output_file', help='full path to output file, if applicable') # parse cl args (allows easy unit testing) args = parser.parse_args(args) if args else parser.parse_args() logging.basicConfig(level=getattr(logging, args.log_level.upper())) sentry_util.configure_sentry() # import the flow module flow = importlib.import_module( '.{}.flow'.format(args.flow), flows.__name__) # look for the flow class, if it exists, otherwise fall back # on the flow module itself. flow_class = getattr(flow, 'Flow', None) if flow_class: flow = flow_class() # execute command args.context = args.context or '{}' _COMMANDS[args.cmd](flow=flow, context=args.context, output_file=args.output_file)