"""Configuration and marshaling of IngestionFeed statuses.""" import datetime from dateutil.relativedelta import relativedelta from flask import request from flask_restful import fields from flask_restful import marshal_with from flask_restful import Resource from feed_status import config from feed_status import util from feed_status.models import feed_monitor, ingestion_feed from feed_status.models.orm import feed_status as dynamo_feed_status ingestion_feed_status_fields = { 'data': fields.List( fields.Nested({ 'delivery_date': fields.String, 'feed_id': fields.String(attribute='feed_id'), 'feed_name': fields.String(attribute='feed_name'), 'metrics': fields.Raw, 'status': fields.Raw, 'status_details': fields.Raw, })), 'dates': fields.List(fields.String) } ingestion_feed_comments_fields = { 'feed_id': fields.String, 'date': fields.String, 'comment': fields.Raw } ingestion_feed_comment_list_fields = {'comments': fields.Raw} class IngestionFeedStatus(Resource): """Provides response for /ingestion_feed//status.""" @marshal_with(ingestion_feed_status_fields) def get(self, feed_id, date=None): """Marshal specific IngestionFeed status. Args: feed_id (str): specific feed of interest. date (str): specific date of interest. Returns: dict: json payload for feed status. """ days_count = config.DASHBOARD_DISPLAYED_DAYS if date: to_date = datetime.date.fromisoformat(date) else: to_date = datetime.date.today() delta = datetime.timedelta(days=days_count) from_date = to_date - delta if feed_id == 'all': ingestion_feeds = ingestion_feed.get_active_feeds() elif feed_id == 'youtube_bulk': ingestion_feeds = ingestion_feed.get_active_youtube_bulk_feeds() elif feed_id == 'youtube_bulk_sme': ingestion_feeds = \ ingestion_feed.get_active_youtube_bulk_sme_feeds() elif feed_id == 'marketshare': ingestion_feeds = ingestion_feed.get_active_marketshare_feeds() delta = datetime.timedelta(days=config.MARKETSHARE_DISPLAYED_DAYS) from_date = to_date - delta elif feed_id == 'monthly': ingestion_feeds = ingestion_feed.get_active_monthly_feeds() delta = datetime.timedelta(days=config.MONTHLY_DISPLAYED_DAYS) from_date = to_date - delta else: ingestion_feeds = [feed_monitor.get_feed_by_id( feed_id, ingestion_only=True)] for feed in ingestion_feeds: if feed_id in ('marketshare', 'monthly'): dates = util.generate_default_month_date_dict_range( from_date, to_date) else: dates = util.generate_default_date_dict_range( from_date, to_date) ingestion_history = dynamo_feed_status.get_ingestion_history( feed.feed_id, from_date.isoformat(), to_date.isoformat() ) feed.status = build_status_from_dynamo_items( dates, ingestion_history ) feed.status_details = build_status_details_from_dynamo_items( dates, ingestion_history ) feed.metrics = feed_monitor.get_metric_status(feed, feed.status) dates = list(dates.keys()) return {'data': ingestion_feeds, 'dates': dates} def build_status_from_dynamo_items(dates_dict, items): """Map ingestion history to status. Args: dates_dict (dict): dict of date => '--'. items (list): dynamodb. Returns: dict: date => status. """ result_dict = dates_dict.copy() for item in items: if 'status' not in item: continue date = item['date'].split('_').pop() if date not in dates_dict: continue status = item['status'].lower() result_dict[date] = status return result_dict def build_status_details_from_dynamo_items(dates_dict, items): """Map ingestion history to status. Args: dates_dict (dict): dict of date => '--'. items (list): dynamodb. Returns: dict: date => status. """ STATUS_DETAILS_PROPERTIES = [ 'status', 'updated_at', 'swf_domain', 'swf_workflow_id', 'swf_run_id', ] result_dict = {k: {} for k in dates_dict} for item in items: date = item['date'].split('_').pop() if date not in dates_dict: continue for property in STATUS_DETAILS_PROPERTIES: if property not in item: continue value = item[property] result_dict[date][property] = value return result_dict class MarketshareIngestionFeedStatus(Resource): """Response for /marketshare_feeds/status.""" # TODO: replace with IngestionFeedStatus @marshal_with(ingestion_feed_status_fields) def get(self, date=None): """Marshal specific IngestionFeed status. Args: date (str): specific date of interest. Returns: dict: json payload for feed status. """ days_count = config.DASHBOARD_DISPLAYED_MONTHS if date: to_date = datetime.date.fromisoformat(date) else: to_date = datetime.date.today() from_date = to_date - relativedelta(months=days_count) ingestion_feeds = ingestion_feed.get_active_marketshare_feeds() all_dates = set() for feed in ingestion_feeds: feed.status = ( ingestion_feed.get_monthly_feed_ingestion_history( feed=feed, from_date=from_date, to_date=to_date, ) ) feed.metrics = feed_monitor.get_metric_status(feed, feed.status) all_dates.update(feed.status.keys()) dates = util.generate_default_month_date_dict_range( from_date, to_date) all_dates.update(dates.keys()) ingestion_feeds = util.set_date_range_for_feeds( ingestion_feeds, all_dates) return { 'data': ingestion_feeds, 'dates': sorted(list(all_dates), reverse=True), } class MonthlyIngestionFeedStatus(Resource): """Response for /monthly_feeds/status.""" # TODO: replace with IngestionFeedStatus @marshal_with(ingestion_feed_status_fields) def get(self, date=None): """Marshal specific IngestionFeed status. Args: date (str): specific date of interest. Returns: dict: json payload for feed status. """ days_count = config.DASHBOARD_DISPLAYED_MONTHS if date: to_date = datetime.date.fromisoformat(date) else: to_date = datetime.date.today() from_date = to_date - relativedelta(months=days_count) ingestion_feeds = ingestion_feed.get_active_monthly_feeds() all_dates = set() for feed in ingestion_feeds: feed.status = ( ingestion_feed.get_monthly_feed_ingestion_history( feed=feed, from_date=from_date, to_date=to_date, ) ) feed.metrics = feed_monitor.get_metric_status(feed, feed.status) all_dates.update(feed.status.keys()) dates = util.generate_default_month_date_dict_range( from_date, to_date) all_dates.update(dates.keys()) ingestion_feeds = util.set_date_range_for_feeds( ingestion_feeds, all_dates) return { 'data': ingestion_feeds, 'dates': sorted(list(all_dates), reverse=True), } class IngestionFeedComment(Resource): """Provides response for /ingestion_feed/comment//.""" @marshal_with(ingestion_feed_comments_fields) def get(self, feed_id, date): """Get comment for choosen date. Args: feed_id (str): specific feed of interest. date (str): date of comment Returns: dict: json payload for feed comments. """ comment = dynamo_feed_status.get_ingestion_comment(feed_id, date) if not comment: return {'message': 'comment not found'}, 404 return { 'feed_id': comment['feed_name'], 'date': comment['date'], 'comment': comment['comment'] }, 200 def post(self, feed_id, date): """Create comment for the date. Args: feed_id (str): specific feed of interest. date (str): date of comment Returns: dict: json payload for feed comments. """ body = request.json new_comment = body.get('comment') comment = dynamo_feed_status.save_ingestion_comment( feed_id, date, new_comment) if not comment: return {'message': 'comment not found'}, 404 return { 'feed_id': comment['feed_name'], 'date': comment['date'], 'comment': comment['comment'] }, 200 def delete(self, feed_id, date): """Delete comment for the date. Args: feed_id (str): specific feed of interest. date (str): date of comment Returns: dict: feed comment delete_status. """ delete_status = dynamo_feed_status.delete_ingestion_comment( feed_id, date) if not delete_status: return {'message': 'comment not found'}, 404 return delete_status, 200 class IngestionFeedCommentList(Resource): """Provides response for /ingestion_feed/comment_list.""" @marshal_with(ingestion_feed_comment_list_fields) def get(self): """Get all comments. Returns: dict: json payload of feed comments. """ # start_date (str): starting date of displayed feed statuses start_date = util.get_start_date() comments = dynamo_feed_status.get_ingestion_comment_list(start_date) return {'comments': comments}, 200