"""Garcon tasks and utils for Feed Ingestion workflows.""" from datetime import date as date_module import sys from decorator import decorator from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.util import task_status STOP_RESPONSE = {'stop': True} def _resolve_arg_values(func, args, feed_name, required_arg_names): """Dynamically resolve argument values. Takes function arguments and resolves required arguments' values. If `feed_name` is not specified we will try to resolve it from arguments as well. Function arguments should match `required_arg_names`, otherwise exception will be raised. Args: func (func): Function. args (tuple): Function arguments. feed_name (str): Feed name. required_arg_names (list): Required arguments list. Returns: list: List with resolved values. """ if feed_name: resolved_values = [feed_name] else: required_arg_names.append('feed_name') resolved_values = [] orig_func_arg_names = ( func.__code__.co_varnames[:func.__code__.co_argcount]) for req_arg_name in reversed(required_arg_names): if req_arg_name in orig_func_arg_names: value = args[orig_func_arg_names.index(req_arg_name)] resolved_values.insert(0, value) else: raise Exception('Argument not provided: %s', req_arg_name) return resolved_values def assert_valid_str_bool(value): """Assert that value is a valid string boolean. It should be either 'True' or 'False'. Args: value (str): Value. Returns: str: validated value. """ if value not in ['True', 'False']: raise ValueError(f'Invalid boolean value: "{value}"') return value def check_status(feed_name=None, task_id=None): """Check DynamoDB status. We expect that some of the wrapped function args are: activity and date. If `feed_name` is not specified we will try to resolve it dynamically from arguments as well. The names should be exactly: activity, date, feed_name. Otherwise an exception will be raised. Args: feed_name (str): Feed name. task_id (str): Task id. Returns: func: Decorated task function. """ @decorator def wrapper(orig_func, *args, **kwargs): resolved_values = _resolve_arg_values( orig_func, args, feed_name, ['activity', 'date']) activity, date, effect_feed_name = resolved_values effect_task_id = task_id or orig_func.__name__ if task_status.is_completed_task( effect_feed_name, date, effect_task_id): activity.logger.info( 'feed: {feed_name} task: {task_id} ' 'date: {date} status: COMPLETE'.format( feed_name=effect_feed_name, task_id=effect_task_id, date=date)) return result = orig_func(*args, **kwargs) if result and result.get('stop') is True: return result else: task_status.mark_completed_task( effect_feed_name, date, effect_task_id) activity.logger.info( 'feed: {feed_name} task: {task_id} ' 'date: {date} status: COMPLETE'.format( feed_name=effect_feed_name, task_id=effect_task_id, date=date)) return result return wrapper def check_ingested_status( feed_name=None, skip_check=getattr(sys, '_called_from_test', False)): """Check INGESTED status. Check if flow for some date has been already ingested. Args: feed_name (str): Feed name. skip_check (bool): Skip check. Returns: func: Decorated task function. """ @decorator def wrapper(orig_func, *args, **kwargs): if skip_check: return orig_func(*args, **kwargs) date, effect_feed_name = _resolve_arg_values( orig_func, args, feed_name, ['date']) date = date or date_module.today().strftime('%Y-%m-%d') overall_status = garcon_feed_status.get_overall_status( effect_feed_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return STOP_RESPONSE return orig_func(*args, **kwargs) return wrapper