"""CLI functions to run decider, worker and exec processes.""" import argparse import os from datetime import datetime from datetime import timedelta import importlib import json import logging import time import boto3 from botocore.config import Config from garcon import activity from garcon import decider from snowflake_etl import flows from snowflake_etl.conf import config as cfg logger = logging.getLogger('cli') AWS_REGION = os.environ.get('AWS_REGION', os.environ.get( 'AWS_DEFAULT_REGION', 'us-east-1')) # common boto3 client config BOTO3_CONFIG = Config( region_name=AWS_REGION, ) def check_and_wait_until_complete(domain, workflow_id): """Wait until worfklow is complete. Args: domain (str): SWF domain. workflow_id (str): Workflow ID, e.g. 'production.fact_analytics- sync_incremental-into-prod.facts-2017-11-06T01_00_22- 2017-11-06T12_27_34'. """ client = boto3.client('swf', region_name='us-east-1') while client.count_open_workflow_executions( domain=domain, startTimeFilter={ 'oldestDate': (datetime.now() - timedelta(days=2)), }, executionFilter={ 'workflowId': workflow_id, })['count'] > 0: logger.info( 'Workflow {} is still running, wait another 20 seconds'.format( workflow_id)) time.sleep(20) def execute_flow(flow, context, output_file=None, wait_until_complete=False, **kwargs): """Launch the workflow execution. Args: flow (module): Garcon flow module. output_file (str): Full path to output file. wait_until_complete (bool): If True, not return until workflow is complete. context (str): Initial context parsed from JSON. kwargs: Extensible call API. """ context_dict = json.loads(context) context_dict['sfdb_params'] = cfg.merge_configs( cfg.SF_PARAMS, context_dict.get('sfdb_params_override')) context = json.dumps(context_dict) context_dict = json.loads(context) context_dict['sfdb_params'] = cfg.merge_configs( cfg.SF_PARAMS, context_dict.get('sfdb_params_override')) start_kwargs = dict( domain=flow.domain, workflowId=flow.workflow_id(json.loads(context)), workflowType={ 'name': flow.name, 'version': flow.version, }, taskList={ 'name': flow.name, }, input=json.dumps(context_dict), childPolicy='TERMINATE', ) swf = boto3.client('swf', config=BOTO3_CONFIG) execution = swf.start_workflow_execution(**start_kwargs) run_id = execution['runId'] if output_file: with open(output_file, 'w') as out_file: start_kwargs.update(run_id=run_id) json.dump(start_kwargs, out_file) if wait_until_complete: check_and_wait_until_complete( flow.domain, workflow_id=start_kwargs['workflow_id']) return execution def run_decider(flow, **kwargs): """Launch the SWF decider process. Args: flow (module): Garcon flow module. kwargs: Extensible call API. """ worker = decider.DeciderWorker(flow) while True: worker.run() time.sleep(1) def run_activity_worker(flow, **kwargs): """Launch the activity worker process. Args: flow (module): Garcon flow module. kwargs: Extensible call API. """ worker = activity.ActivityWorker(flow) worker.run() _COMMANDS = {'exec': execute_flow, 'decider': run_decider, 'worker': run_activity_worker} def garcon(*args): """Main entry point for the Garcon command line integration.""" parser = argparse.ArgumentParser(description='Garcon command line util') parser.add_argument('cmd', choices=_COMMANDS.keys(), help='garcon command') parser.add_argument('flow', choices=flows.__all__, help='name of the flow') parser.add_argument('-c', '--context', help='initial context [json]') parser.add_argument('-o', '--output_file', help='full path to output file, if applicable') parser.add_argument( '--wait-until-complete', help='Not return until workflow is complete', action='store_true') # parse cl args (allows easy unit testing) args = parser.parse_args(args) if args else parser.parse_args() # import the flow module flow = importlib.import_module( '.{}.flow'.format(args.flow), flows.__name__) # look for the flow class, if it exists, otherwise fall back # on the flow module itself. flow_class = getattr(flow, 'Flow', None) if flow_class: flow = flow_class() # execute command args.context = args.context or '{}' _COMMANDS[args.cmd]( flow=flow, context=args.context, output_file=args.output_file, wait_until_complete=args.wait_until_complete)