"""Tasks for handling overall feed status in DynamoDB.""" import datetime from garcon import task from garcon_contrib.dynamo_feed_status import garcon_feed_status from feed_ingestion.util import task_status from feed_ingestion.util.log_status import log_feed_ingestion_status # todo: 0 usages. Remove unused code @task.decorate(timeout=1000) def advance_overall_status(activity, date, feed_name): """Determine and set the next overall status. Statuses of each file and the previous overall status are being checked in between tasks. If the previous task was successful, this task will be called to advance the status of the workflow. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYYMMDD). feed_name (str): Name of the feed. """ current_status = garcon_feed_status.get_overall_status(feed_name, date) activity.logger.debug('current status: ' + str(current_status)) if current_status != garcon_feed_status.STATUS_INGESTED: if current_status == garcon_feed_status.STATUS_DOWNLOADED: garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_POPULATED_RAW_TABLE) activity.logger.debug( 'new status: ' + str( garcon_feed_status.STATUS_POPULATED_RAW_TABLE)) elif current_status == garcon_feed_status.STATUS_POPULATED_RAW_TABLE: garcon_feed_status.set_overall_status( feed_name, date, garcon_feed_status.STATUS_INGESTED) activity.logger.debug( 'new status: ' + str(garcon_feed_status.STATUS_INGESTED)) else: garcon_feed_status.set_overall_status(feed_name, date) activity.logger.debug('new status else') def set_overall_status_enhanced(feed_name, date, status, activity): """Set the overall status of a feed with additional information. Adds attributes: updated_at with iso datetime format 'YYYY-MM-DDTHH:MM:SSZ' (UTC timezone) swf_domain, swf_run_id, swf_workflow_id with SWF execution information. Args: feed_name (str): Name of the feed. date (str): Reporting date (YYYYM-MM-DD). status (str): Status constant in util.garcon_feed_status. activity (ActivityWorker): The activity worker. """ attributes = {} # save updated_at as iso string datetime now = datetime.datetime.utcnow().replace(microsecond=0) attributes['updated_at'] = f'{now.isoformat()}Z' # add SWF Execution information context_to_item_attribute_map = { 'execution.domain': 'swf_domain', 'execution.run_id': 'swf_run_id', 'execution.workflow_id': 'swf_workflow_id', } context = getattr(activity, 'context', {}) for context_key, attribute_key in context_to_item_attribute_map.items(): if context_key in context: attributes[attribute_key] = activity.context[context_key] garcon_feed_status.set_overall_status( feed_name, date, status, attributes=attributes) log_feed_ingestion_status(activity, feed_name, date, status) activity.logger.info( f'Setting status for feed: {feed_name} ' f'with date: {date} ' f'to: {status}, via a task'.format( feed_name=feed_name, date=date, status=status)) @task.decorate(timeout=1000) def set_overall_status( activity, date, feed_name, status, set_status_once=None): """Explicitly set the overall status of a feed. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYYM-MM-DD). feed_name (str): Name of the feed. status (str): Status constant in util.garcon_feed_status. set_status_once (bool): If True, only set status if it has not been set for the feed_name and date previously (defaults to False) """ set_status_once = set_status_once or False task_id = 'set_overall_status_{}'.format(status) if set_status_once and task_status.is_completed_task( feed_name, date, task_id): activity.logger.info( 'status {status} previously updated ' 'for {feed_name} - {date}, skipping'.format( feed_name=feed_name, date=date, status=status)) return set_overall_status_enhanced( feed_name, date, status, activity=activity) task_status.mark_completed_task(feed_name, date, task_id) @task.decorate(timeout=1000) def delete_overall_status(activity, date, feed_name): """Delete DynamoDB status for feed. Args: activity (ActivityWorker): The activity worker. date (str): Reporting date (YYYYM-MM-DD). feed_name (str): Name of the feed. """ garcon_feed_status.delete_status(feed_name, date) activity.logger.info( 'Deleting status for feed: {feed_name} ' 'with date: {date}'.format(feed_name=feed_name, date=date)) @task.decorate(timeout=1000) def check_feed_status( activity, feed_name, date, reload, licensor=None, report=None): """Check and reset feed status if it is needed. Args: activity (ActivityWorker): The Garcon activity worker. feed_name (str): Feed name of workflow execution for status updates. date (str): Reporting date (YYYY-MM-DD). reload (str or None): If 'True' delete all feed statuses in DynamoDB. licensor (str): The licensor. report (str): The report. Returns: dict: dict or STOP_RESPONSE. """ # Build feed name with only non-None components overall_feed_name = '_'.join(filter(None, [feed_name, licensor, report])) if reload == 'True': activity.logger.info('Delete status for feed: {} {} '.format( overall_feed_name, date)) garcon_feed_status.delete_status(overall_feed_name, date) else: overall_status = garcon_feed_status.get_overall_status( overall_feed_name, date) if overall_status == garcon_feed_status.STATUS_INGESTED: return {'stop': True} return {'feed_name': overall_feed_name}