""" ETL FLow Command Line. All of the general command line bootstrapping is here. It calls hooks from each ETL flow.cli module to complete the argparse.ArgumentParser initialization. """ import argparse import importlib import inspect import logging import uuid import flows from flows import log _log_levels = ['debug', 'info', 'warning', 'error', 'critical'] # These are the sub-commands for the cli interface. Each of these sub-commands # will trigger the flows' respective argument init function. See the # init_flow_sub_commands function for more detailed information. _sub_commands = [ {'command': 'execute', 'help': 'Start an ETL process'}, {'command': 'decider', 'help': "Start an ETL's decider process"}, {'command': 'worker', 'help': "Start an ETL's worker process"}, ] def get_flows_cli(): """Lookup and get the ETL flows' cli modules. Returns: dict: imported flow cli modules. """ modules = {} for flow in flows.__all__: module = 'flows.{flow}.cli'.format(flow=flow) modules[flow] = importlib.import_module(module) return modules def init_flow_sub_commands(subparser, flows, sub_commands): """Initialize ETL flow command line arguments. Each flow has a sub-command to the cli interface with a unique set of arguments. This iterates through all flows and calls the sub-command initialization hook, with the format: flow_module.cli.init_{sub_command}_parser This allows each flow to keep their command configuration isolated as well as keep argument configurations from getting mixed up. Args: subparser (argparse.ArgumentParser): argument parser object for config. flows (dict): imported flow_module.cli modules (names as keys). sub_commands (list): list of sub-commands to initialize, along with the general sub-command help string. """ for cmd in sub_commands: cmd_parser = subparser.add_parser(cmd['command'], help=cmd['help']) cmd_subparser = cmd_parser.add_subparsers( dest='sub_command_flow', help='Flows') for flow in flows: init_method_name = 'init_{cmd}_parser'.format(cmd=cmd['command']) if hasattr(flows[flow], init_method_name): flow_subparser = cmd_subparser.add_parser(flow) getattr(flows[flow], init_method_name)(flow_subparser) def main(): """Do the thing, which means setup cli arguments and run.""" parser = argparse.ArgumentParser( description='Start a Film Transparency ETL component') parser.add_argument( '-l', '--log-level', help='Log level', choices=_log_levels, default='error') subparsers = parser.add_subparsers(dest='sub_command', help='Sub-commands') flows = get_flows_cli() init_flow_sub_commands(subparsers, flows, _sub_commands) args = parser.parse_args() log.set_logger(getattr(logging, args.log_level.upper())) if not args.sub_command: parser.parse_args(['--help']) return elif not args.sub_command_flow: parser.parse_args([args.sub_command, '--help']) return correlation_id = str(uuid.uuid1()) run_method_flow = flows[args.sub_command_flow] run_method_name = 'run_{cmd}'.format(cmd=args.sub_command) run_method_params = vars(args) del run_method_params['sub_command'] del run_method_params['sub_command_flow'] del run_method_params['log_level'] run_method = getattr(run_method_flow, run_method_name) run_method_parameters = inspect.signature(run_method).parameters if 'correlation_id' in run_method_parameters: run_method_params['correlation_id'] = correlation_id run_method(**run_method_params)