"""Utility functions for executing flows.""" import argparse import datetime from enum import Enum import importlib import json import logging import os from pathlib import Path import re import subprocess import sys import time from typing import Callable, Iterable from garcon_contrib.dynamo_feed_status import garcon_feed_status import jinja2 from feed_ingestion import cli, flows from feed_ingestion.util import context_util, dates_util from feed_ingestion.util.aws import swf as swf_util from feed_ingestion.util.dates_util import Weekday logger = logging.getLogger(__name__) class CustomArgumentParser(argparse.ArgumentParser): """Custom ArgumentParser which allows to set default values from ENV.""" def __init__(self, **kwargs): """Init custom parser.""" # disable auto-abbreviations for arguments # kwargs['allow_abbrev'] = False self.env_to_action = {} self.context_actions = [] self.args = [] super().__init__(**kwargs) class ContextAction: """Base class for context actions.""" def update_context(self, context, namespace): """Fill context from namespace.""" raise NotImplementedError() class ContextActionMap(ContextAction): """Map namespace value to context key.""" def __init__( self, namespace_dest: str, context_key: str, formatter: Callable = None, skip_if_none: bool = False, ): """Init context action.""" self.context_key = context_key self.namespace_dest = namespace_dest if not formatter: formatter = lambda x: x # noqa:E731 self.formatter = formatter self.skip_if_none = skip_if_none def update_context(self, context, namespace): """Fill context from namespace.""" value = getattr(namespace, self.namespace_dest) value_formatted = self.formatter(value) if value_formatted is None and self.skip_if_none: return context[self.context_key] = value_formatted def add_arg( self, name: str, flags: list = None, env_var: str = None, context_key: str = None, context_value_formatter: Callable = None, default=None, required=False, help: str = None, # noqa: A002 **kwargs, ): """Add argument to the parser. It supports setting the value from the environment variable. Args: self: (CustomArgumentParser) The parser name: (str) The name of the argument. flags: (str) [optional] command line flags for the argument. By default, will be used "--name" env_var: (str) [optional] The name of the environment variable which can define the value of the arg context_key: (str) Optional key name in context. Value of this will be added to '_context' property in args Namespace context_value_formatter: optional callable to convert argv value to context value required: (bool) True if value required to be set default: value used if other not set help: (optional) help message **kwargs: (dict) Additional keyword arguments to add_arg Returns: None """ if not re.match(r'^[\w_]+$', name): raise ValueError(f'Invalid argument name: {name}') if default and required: raise ValueError('Cannot have both default and required set') existing_env_var = self.env_to_action.get(env_var) if existing_env_var: raise ValueError(f'Env var "{env_var}" ' f'already used by {existing_env_var}') arg = dict( name=name, flags=flags, env_var=env_var, context_key=context_key, default=default, required=required, help=help, **kwargs ) self.args.append(arg) if not flags: flags = [f'--{name.replace("_", "-")}'] kwargs['dest'] = name help = help or f'"{name}" argument.' # noqa: A001 if context_key: help += f' Will be passed to context with name: "{context_key}".' if default: help += f' (Default: "{default}").' kwargs['default'] = default if env_var: help += f' Can also be set by env var "{env_var}".' kwargs['help'] = help kwargs['required'] = required action = self.add_argument( *flags, **kwargs ) if env_var is not None: self.env_to_action[env_var] = action if context_key: context_action = self.ContextActionMap( namespace_dest=name, context_key=context_key, formatter=context_value_formatter, skip_if_none=not required ) self.context_actions.append(context_action) def _get_env_args(self): argv = [] for env_var, action in self.env_to_action.items(): value = os.getenv(env_var) if value is not None: flag = action.option_strings[0] argv_entry = f'{flag}={value}' argv.append(argv_entry) return argv def parse_args(self, args=None, namespace=None): """Parse args and set default values from ENV.""" assert args is not None, 'args should be set' env_vars = self._get_env_args() argv_full = env_vars + args namespace = super().parse_args(argv_full, namespace) # fill _context context = {} for context_action in self.context_actions: context_action.update_context(context, namespace) if context: setattr(namespace, '_context', context) return namespace class FlowExecBase(): """Base class for exec.""" flow_name = None DEFAULT_CONTEXT = '{}' def __init__(self): """Init exec.""" self.args = None def prepare_parser(self) -> CustomArgumentParser: """Prepare parser for the flow exec.""" self.parser = self.create_parser() self.add_args() return self.parser def create_parser(self): """Create empty arg parser.""" parser = CustomArgumentParser( description=f'Executor for SWF flow "{self.flow_name}". ' 'Use single argument "jenkinsfile" ' 'to generate Jenkinsfile near exec.py' ) return parser def add_args(self): """Add args to parser.""" add_verbose_arg(self.parser) add_context_date_arg(self.parser) add_reload_arg(self.parser) add_raw_context_arg(self.parser, default=self.DEFAULT_CONTEXT) add_wait_until_complete_arg(self.parser) def generate_dates(self) -> Iterable[datetime.date]: """Generate dates for execution.""" context_date = getattr(self.args, 'context_date', None) if context_date: logger.info(f'Using single date execution: {context_date}') yield context_date def generate_contexts(self) -> Iterable[dict]: """Generate contexts variants for execution.""" context = getattr(self.args, 'context', {}) yield context def generate_jenkinsfile(self): """Generate Jenkinsfile.""" logger.info(f'Generating Jenkinsfile for {self.flow_name}') parser_args = self.parser.args jinja_env = jinja2.Environment() template_dir = Path(__file__).parent template_filename = 'Jenkinsfile.j2' jinja_env.loader = jinja2.FileSystemLoader( template_dir) logger.info(f'Using template "{template_dir / template_filename}"') template = jinja_env.get_template(template_filename) jenkinsfile = template.render( flow_name=self.flow_name, arguments=parser_args, ) import __main__ main_path = Path(__main__.__file__) path = main_path.parent / 'Jenkinsfile' logger.info(f'Going to generate Jenkinsfile at "{path}"') if main_path.name != 'exec.py': raise ValueError(f'Only generate when run from exec.py ' f'(actual run from {main_path})') with path.open('w') as f: f.write(jenkinsfile) logger.info('Jenkinsfile was saved.') def execute(self): """Execute a flow with the given arguments. According to configuration from args there will be triggers of `garcon exec` with context. Returns: None """ contexts = list(self.generate_contexts()) dates = list(self.generate_dates()) logger.info(f'Having {len(contexts)} context(s) to execute.') logger.info(f'Dates to execute: {[str(d) for d in dates]}') if len(contexts) * len(dates) == 0: raise ValueError('Nothing to execute') errors = [] args_context = getattr(self.args, '_context', {}) for date in dates: for context in contexts: try: execution_context = {**args_context, **context} self.execute_context( date=date, context=execution_context, ) except subprocess.SubprocessError as e: errors.append(e) if errors: raise Exception(f'Errors: {errors}') def parse_args(self, argv): """Parse args.""" self.args = self.parser.parse_args(args=argv) def main(self): """Execute the flow.""" argv = sys.argv[1:] verbose = any( arg in argv for arg in ['-v', '--verbose'] ) self.configure_logging(verbose=verbose) self.parser = self.prepare_parser() if len(argv) > 0 and argv[0] == 'jenkinsfile': self.generate_jenkinsfile() else: self.parse_args(argv=argv) self.execute() def _get_flow_instance(self, flow_name): if not self.flow_name: raise ValueError('flow_name should be set') flow = importlib.import_module( '.{}.flow'.format(flow_name), flows.__name__ ) # look for the flow class, if it exists, otherwise fall back # on the flow module itself. flow_class = getattr(flow, 'Flow', None) if flow_class: flow = flow_class() return flow def execute_context( self, context: dict, date: datetime.date, ): """Execute a flow for a single date.""" context['context_date'] = date.strftime('%Y-%m-%d') context_str = json.dumps(context) if not self.flow_name: raise ValueError('flow_name should be set') logger.info(f'Executing "{self.flow_name}" ' f'with context: {context_str}') flow = self._get_flow_instance(self.flow_name) cli.execute_flow(flow, context_str) def configure_logging(self, verbose=False): """Configure exec for a flow.""" level = logging.DEBUG if verbose else logging.INFO logging.basicConfig( level=level, format='%(asctime)s %(levelname)s %(message)s', ) logger.info( f'Configured exec for {self.flow_name}. ' f'Verbose output: {verbose}' ) class LicensorMixin(FlowExecBase): """Add licensor arg.""" LICENSORS = [] LICENSOR_USE_ALL = True def __arguments(self): choices = list(self.LICENSORS) if self.LICENSOR_USE_ALL: choices = ['ALL'] + choices add_licensor_arg( parser=self.parser, choices=choices, ) def generate_contexts(self) -> Iterable[dict]: """Generate contexts.""" if not self.args.licensor: yield from super().generate_contexts() return if self.args.licensor == 'ALL': licensors = self.LICENSORS else: licensors = [self.args.licensor] for parent_context in super().generate_contexts(): for licensor in licensors: yield {**parent_context, 'licensor': licensor} def add_args(self): """Add args to parser.""" self.__arguments() super().add_args() class CountryMixin(FlowExecBase): """Add licensor arg.""" COUNTRIES = [] COUNTRIES_USE_ALL = True def __arguments(self): choices = list(self.COUNTRIES) if self.COUNTRIES_USE_ALL: choices = ['ALL'] + choices self.parser.add_arg( 'country', env_var='COUNTRY', context_key='country', help='Country.', required=True, choices=choices, ) def generate_contexts(self) -> Iterable[dict]: """Generate contexts.""" if not self.args.country: yield from super().generate_contexts() return if self.args.country == 'ALL': countries = self.COUNTRIES else: countries = [self.args.country] for parent_context in super().generate_contexts(): for country in countries: yield {**parent_context, 'country': country} def add_args(self): """Add args to parser.""" self.__arguments() super().add_args() class ReportMixin(FlowExecBase): """Add report arg.""" REPORTS = [] REPORT_USE_ALL = True REPORT_CONTEXT_KEY = 'report_name' def __arguments(self): choices = list(self.REPORTS) if self.REPORT_USE_ALL: choices = ['ALL'] + choices add_report_arg( parser=self.parser, choices=choices, context_key=self.REPORT_CONTEXT_KEY, ) def generate_contexts(self) -> Iterable[dict]: """Generate contexts.""" if not self.args.report: yield from super().generate_contexts() return if self.args.report == 'ALL': reports = self.REPORTS else: reports = [self.args.report] for parent_context in super().generate_contexts(): for report in reports: yield {**parent_context, self.REPORT_CONTEXT_KEY: report} def add_args(self): """Add args to parser.""" self.__arguments() super().add_args() class CheckStatusMixin(): """Add functionality to skip completed executions.""" DEFAULT_EXPECTED_STATUS = garcon_feed_status.STATUS_INGESTED def add_args(self): """Add args to parser.""" add_check_status_args( self.parser, default_expected_status=self.DEFAULT_EXPECTED_STATUS, ) super().add_args() @staticmethod def get_status_for_flow_context_and_date( flow: str, context: dict, date: datetime.date): """Get the feed status for the context from DynamoDB.""" flow_entity = flows.get_flow(flow) date_str = date.strftime('%Y-%m-%d') feed_name = flow_entity.contextified_feed_name(context) item = garcon_feed_status._get_item(feed_name, date_str) status = item.get('status') if item else None logger.info(f'Status for "{feed_name}" at {date} is "{status}"') return status def is_context_completed( self, context: dict, date: datetime.date, ): """Check if the flow was already executed.""" if (is_true_arg(self.args.check_status) and not is_true_arg(self.args.reload)): status = self.get_status_for_flow_context_and_date( flow=self.flow_name, context=context, date=date ) logger.info(f'Actual status is "{status}"' f', expected "{self.args.expected_status}"') return status == self.args.expected_status return False def execute_context(self, context: dict, date: datetime.date): """Execute flow for a single context and date.""" if self.is_context_completed(context=context, date=date): logger.info(f'Already completed. Skipping {context}') return super().execute_context(context=context, date=date) class ConcurrencyControlMixin(FlowExecBase): """Mixin for controlling concurrency of flow executions.""" class Strategy(Enum): """Enum for concurrency strategies.""" def __new__(cls, value, description): """Create new instance of the enum.""" obj = object.__new__(cls) obj._value_ = value obj.description = description return obj SKIP = 'SKIP', 'do not execute contexts which are above limit', IGNORE = 'IGNORE', 'execute all contexts ignoring limit', FAIL = 'FAIL', 'fail execution if limit is reached' WAIT = 'WAIT', 'delay context execution until limit free' DEFAULT_MAX_CONCURRENT_FLOWS = 5 SLEEP_SECONDS = 30 # timeout to wait for execution slot TIMEOUT_SECONDS = 3600 DEFAULT_CONCURRENCY_STRATEGY = Strategy.WAIT @staticmethod def add_concurrency_control_args( parser, default_max_concurrent_flows, default_concurrency_strategy, ): """Add context arguments to the parser.""" parser.add_arg( 'max_concurrent_flows', env_var='MAX_CONCURRENT_FLOWS', type=int, help='Limit of active SWF flow executions.', default=default_max_concurrent_flows, metavar='N', required=False, ) strategy_help = [ f'[{e.value}] - {e.description}.' for e in ConcurrencyControlMixin.Strategy ] parser.add_arg( 'concurrency_strategy', env_var='CONCURRENCY_STRATEGY', help='Action to perform when concurrency limit reached. ' + ' '.join(strategy_help), default=default_concurrency_strategy.value, choices=[e.value for e in ConcurrencyControlMixin.Strategy], ) def add_args(self): """Add args to parser.""" self.add_concurrency_control_args( self.parser, default_max_concurrent_flows=self.DEFAULT_MAX_CONCURRENT_FLOWS, default_concurrency_strategy=self.DEFAULT_CONCURRENCY_STRATEGY, ) super().add_args() def execute_context(self, context: dict, date: datetime.date): """Execute a flow for a single date.""" flow_instance = flows.get_flow(self.flow_name) start_time = datetime.datetime.now() timeout = datetime.timedelta(seconds=self.TIMEOUT_SECONDS) max_concurrent_flows = self.args.max_concurrent_flows concurrency_strategy_str = self.args.concurrency_strategy if not max_concurrent_flows or not concurrency_strategy_str: logger.warning( 'Concurrency control is not configured. Processing without it') return super().execute_context(context=context, date=date) strategy = self.Strategy(concurrency_strategy_str) while datetime.datetime.now() - start_time < timeout: num_flows = swf_util.count_running_workflows_by_type( domain=flow_instance.domain, type_name=flow_instance.name, ) logger.info( f'Number of active flows: {num_flows} ' f' (limit = {max_concurrent_flows})') if num_flows < max_concurrent_flows: logger.info('Concurrency limit check passed.') return super().execute_context(context=context, date=date) if strategy == self.Strategy.SKIP: logger.warning( f'Concurrency limit reached. ' f'Skipping execution for {context}' ) return elif strategy == self.Strategy.IGNORE: logger.warning( f'Concurrency limit reached. ' f'Ignoring and continue execution for {context}' ) return super().execute_context(context=context, date=date) elif strategy == self.Strategy.FAIL: raise RuntimeError( f'Concurrency limit reached. ' f'Failing execution for {context}' ) elif strategy == self.Strategy.WAIT: remaining_timeout = ( timeout - (datetime.datetime.now() - start_time)) logger.info( f'Concurrency limit reached. ' f'Waiting for {self.SLEEP_SECONDS} seconds. ' f'(Timeout in: {remaining_timeout}).' ) time.sleep(self.SLEEP_SECONDS) continue raise ValueError( f'Unexpected state. Concurrency strategy: {strategy}' ) raise TimeoutError(f'FAIL. Execution timeout {timeout} reached.') class ScheduledFlowExec( CheckStatusMixin, ConcurrencyControlMixin, FlowExecBase): """Parent class for all exec scripts for scheduled runs.""" pass class FlowExecDaily(ScheduledFlowExec): """Exec for daily flows.""" def __arguments(self): """Args for this class.""" add_skip_arg( self.parser, default=1, ) add_count_period_arg( self.parser, period='days' ) def add_args(self): """Add args to parser.""" self.__arguments() super().add_args() def generate_dates(self) -> Iterable[datetime.date]: """Generate dates for execution.""" if self.args.days and self.args.context_date: raise ValueError( 'Cannot use both days and context_date arguments.' ) if self.args.context_date: yield from super().generate_dates() return if self.args.days: today = datetime.date.today() logger.info( f'Executing for {self.args.days} day(s), ' f'skipping {self.args.skip} day(s),' f'today is {today}' ) yield from dates_util.generate_dates( skip=self.args.skip, days=self.args.days, initial_date=today ) class FlowExecMonthly(ScheduledFlowExec): """Exec for daily flows.""" def __arguments(self): """Args for this class.""" add_skip_arg( self.parser, default=1, ) add_count_period_arg( self.parser, default=1, period='months' ) def add_args(self): """Add args to parser.""" self.__arguments() super().add_args() def generate_dates(self) -> Iterable[datetime.date]: """Generate dates for execution.""" if self.args.months and self.args.context_date: raise ValueError( 'Cannot use both months and context_date arguments.' ) if self.args.context_date: yield from super().generate_dates() return if self.args.months: today = datetime.date.today() logger.info( f'Executing for {self.args.months} month(s), ' f'skipping {self.args.skip} month(s),' f'today is {today}' ) yield from dates_util.generate_months( skip=self.args.skip, months=self.args.months, initial_date=today ) class FlowExecWeekly(ScheduledFlowExec): """Exec for daily flows.""" def __arguments(self): """Args for this class.""" add_skip_arg( self.parser, default=1, ) add_count_period_arg( self.parser, period='weeks', default=0 ) add_weekday_arg( self.parser, default=None, required=False, ) def add_args(self): """Add args to parser.""" self.__arguments() super().add_args() def generate_dates(self) -> Iterable[datetime.date]: """Generate dates for execution.""" if self.args.weeks and self.args.context_date: raise ValueError( 'Cannot use both weeks and context_date arguments.' ) if self.args.context_date: yield from super().generate_dates() return if self.args.weeks: today = datetime.date.today() weekday = self.args.weekday logger.info( f'Executing for {self.args.weeks} week(s), ' f'skipping {self.args.skip} week(s),' f'today is {today},' f'weekday is {weekday},' ) weekday = weekday.value if weekday else None yield from dates_util.generate_weeks( skip=self.args.skip, weeks=self.args.weeks, initial_date=today, weekday=weekday, ) def add_check_status_args( parser: CustomArgumentParser, default_expected_status, ): """Add check_status and expected_status arguments to the parser.""" add_arg_bool( parser=parser, name='check_status', env_var='CHECK_STATUS', required=False, help='Check status and do not run already completed context.' ) choices = [ garcon_feed_status.STATUS_INGESTED, garcon_feed_status.STATUS_DOWNLOADED, garcon_feed_status.STATUS_POPULATED_RAW_TABLE, ] parser.add_arg( 'expected_status', env_var='EXPECTED_STATUS', help=( 'Value of status which considered as completed.' ), default=default_expected_status, choices=choices, ) def add_count_period_arg( parser: CustomArgumentParser, period: str, default=None, required=False ): """Add period argument to the parser.""" assert period in ['days', 'weeks', 'months'] parser.add_arg( period, env_var=period.upper(), default=default, required=required, type=argparse_type_int_or_none, metavar='N', help=( f'Number of {period} to exec going backward' ) ) def add_skip_arg(parser: CustomArgumentParser, default=None, required=False): """Add skip argument to the parser.""" parser.add_arg( 'skip', env_var='SKIP', type=argparse_type_int_or_none, required=required, default=default, metavar='N', help=( 'Number of periods to skip going backwards.' ) ) def add_weekday_arg( parser: CustomArgumentParser, **kwargs, ): """Add weekday argument to the parser.""" choices = [e.name.capitalize() for e in Weekday] def weekday_type(value): """Validate the weekday string for argparse argument.""" if not value: return None if isinstance(value, Weekday): return value try: return Weekday[value.upper()] except KeyError: raise argparse.ArgumentTypeError( f"Invalid weekday: '{value}'. Expected one of: {choices}" ) kwargs.setdefault('name', 'weekday') kwargs.setdefault('env_var', kwargs['name'].upper()) kwargs['type'] = weekday_type kwargs['choices'] = [weekday for weekday in Weekday] parser.add_arg( **kwargs ) def reload_type(value): """Validate the value for reload.""" if not value: return None try: return context_util.Reload.from_value(value).name.capitalize() except ValueError: raise argparse.ArgumentTypeError( f"Invalid json: '{value}'" ) def add_reload_arg(parser): """Add optional reload argument to the parser.""" choices = [c.name.capitalize() for c in context_util.Reload] parser.add_arg( name='reload', context_key='reload', choices=choices, metavar=' | '.join(choices), type=reload_type, env_var='RELOAD', help='"True" - clear status before start, ' '"Soft" - skip tasks loading raw data, ' '"None" - normal processing', default=None, required=False ) def add_build_dbt_arg(parser): """Add optional reload argument to the parser.""" add_arg_bool( parser=parser, name='build_dbt', context_key='build_dbt', env_var='BUILD_DBT', help='When True it triggers build DBT models', default=False, required=False ) def add_wait_until_complete_arg(parser): """Add optional wait_until_complete argument to the parser.""" add_arg_bool( parser=parser, name='wait_until_complete', context_key='wait_until_complete', env_var='WAIT_UNTIL_COMPLETE', default=False, help='If "True" then wait until finish execution.', required=False ) def argparse_type_date(value): """Validate the date string for argparse argument.""" if not value: return None try: date = datetime.datetime.strptime(value, '%Y-%m-%d').date() return date except ValueError: raise argparse.ArgumentTypeError( f"Invalid date: '{value}'. Expected format: YYYY-MM-DD" ) def argparse_type_int_or_none(value): """Validate the date string for argparse argument.""" if not value: return None try: return int(value) except ValueError: raise argparse.ArgumentTypeError( f"Invalid: '{value}'. Expected integer or empty string." ) def argparse_type_bool(value): """Validate the bool string for argparse argument.""" if str(value).capitalize() == 'True': return 'True' elif str(value).capitalize() == 'False': return 'False' elif not value: return None else: raise argparse.ArgumentTypeError( f"Invalid bool: '{value}'. Expected one of: True,False" ) def add_context_date_arg( parser, context_key='context_date'): """Add context_date argument to the parser.""" formatter = lambda x: x.strftime('%Y-%m-%d') if x else None # noqa:E731 add_arg_date( parser=parser, name='context_date', flags=['--context-date', '--date'], env_var='CONTEXT_DATE', context_key=context_key, context_value_formatter=formatter, required=False, help='Single context date.' ) def add_verbose_arg(parser): """Add verbose argument to the parser.""" parser.add_argument( '-v', '--verbose', action='store_true', default=False, help='Verbose debug output' ) def add_licensor_arg(parser, choices): """Add licensor argument to the parser.""" parser.add_arg( 'licensor', env_var='LICENSOR', context_key='licensor', help='Licensor.', required=True, choices=choices, ) def add_report_arg( parser, choices, name='report', flags=('--report-name', '--report'), context_key='report_name', env_var='REPORT', required=True, help='Report name.', # noqa: A002 **kwargs ): """Add report_name argument to the parser.""" parser.add_arg( name=name, flags=flags, context_key=context_key, env_var=env_var, choices=choices, required=required, help=help, **kwargs ) def json_arg_type(value): """Validate the date string for argparse argument.""" if not value: return {} try: return json.loads(value) except ValueError: raise argparse.ArgumentTypeError( f"Invalid json: '{value}'" ) def add_raw_context_arg(parser, **kwargs): """Add context argument to the parser.""" parser.add_arg( 'context', env_var='CONTEXT', type=json_arg_type, help='Optional context arguments', metavar='JSON', required=False, **kwargs ) def is_true_arg(value): """Check if the value is a true argument.""" return str(value).capitalize() == 'True' def add_arg_bool( parser: CustomArgumentParser, name: str, **kwargs, ): """Add bool argument to the parser.""" kwargs['type'] = argparse_type_bool kwargs['metavar'] = 'True | False' return parser.add_arg( name, **kwargs, ) def add_arg_date( parser: CustomArgumentParser, **kwargs, ): """Add date argument to the parser.""" kwargs['type'] = argparse_type_date kwargs['metavar'] = 'YYYY-MM-DD' return parser.add_arg( **kwargs, )