"""Decorators for ingestion workflows' bootstrap. Usage: @bootstrap.reset_dynamodb_status_on_reload({feed_name}) """ from datetime import date as date_module from functools import wraps from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.tasks import feed_status_tasks def reset_dynamodb_status_on_reload(feed_name): """Set DynamoDB status. Applying this decorator to delete specific status item if parameter 'reload' is set to 'True'. Args: feed_name (str): Feed name. Returns: func: Decorated bootstrap function. """ def inner_wrapper(func): @wraps(func) def wrapper(activity, date, reload=None, dw_config=None, test=None): if reload == 'True': activity.logger.info('Delete status for feed: {} {} '.format( feed_name, date)) garcon_feed_status.delete_status(feed_name, date) if test == 'True': # for integration testing purposes return func(activity, date, dw_config, test) else: return func(activity, date, dw_config) return wrapper return inner_wrapper def block_non_sequential_then_reset_on_reload(feed_name): """Guard a feed against non-sequential (past-date) processing. Stops the workflow when the requested date is older than the latest INGESTED date for the feed. The check runs regardless of the reload flag, so it also catches a past date injected via Jenkins. On a blocked date the DynamoDB status is left untouched (monitor history preserved). On an allowed reload the status is deleted as before, which clears completed-task markers so the run actually reprocesses. Args: feed_name (str): Feed name. Returns: func: Decorated bootstrap function. """ def inner_wrapper(func): @wraps(func) def wrapper(activity, date, reload=None, dw_config=None): date = date or date_module.today().strftime('%Y-%m-%d') stop = feed_status_tasks.has_newer_ingested_date(feed_name, date) if stop: activity.logger.warning( 'Blocking %s for %s: a newer date is already INGESTED; ' 'refusing non-sequential / past-date load.', feed_name, date) elif reload == 'True': activity.logger.info( 'Reload requested: deleting status for %s %s.', feed_name, date) garcon_feed_status.delete_status(feed_name, date) return func(activity, date, dw_config, stop=stop) return wrapper return inner_wrapper