"""FeedMonitor class and finder method.""" import collections from datetime import date from datetime import datetime from datetime import timedelta import json from typing import Dict from typing import List from typing import Optional from typing import OrderedDict from typing import Union from feed_status import config from feed_status import flows from feed_status.models import activity_detector_feed as ad_feed from feed_status.models import delphi_feed from feed_status.models import ingestion_feed from feed_status.models import outgoing_feed from feed_status.models.activity_detector_feed import ActivityDetectorFeed from feed_status.models.delphi_feed import DelphiFeed from feed_status.models.ingestion_feed import IngestionFeed from feed_status.models.outgoing_feed import OutgoingFeed METRIC_FRESH = 'current_status' METRIC_STABLE = 'stability' METRIC_RECENT_SYNC = 'most_recent_sync' METRIC_STATUS_OK = 'ok' METRIC_STATUS_CRITICAL = 'critical' METRIC_STATUS_PENDING = 'pending' _METRIC_CATEGORY = { METRIC_FRESH: 'freshness', METRIC_RECENT_SYNC: 'freshness', METRIC_STABLE: 'stability', } class FeedMonitor(object): """FeedMonitor class to store configuration about each feed.""" def __init__(self, feed_id: str, metric_name: str, threshold: int, threshold_time: Optional[str], only_business_days: bool, success_status: str) -> None: """Initialize class with attributes. Args: feed_id: identifier for feed. metric_name: name of metric being monitored. threshold: recent days excluded from health metric. threshold_time: 24hr time of day in "%H:%M" format. Used to determine whether current day included in threshold. only_business_days: whether monitor considers business days in evaluation of metric status. success_status: status which indicates successful ETL completion. """ self.feed_id = feed_id self.metric_name = metric_name self.threshold = threshold if threshold_time is None: threshold_time = '01:00' self.threshold_time = threshold_time self.only_business_days = only_business_days self.success_status = success_status def match(self, **kwargs) -> bool: """Instance of FeedMonitor must contain all provided attributes. Args: kwargs: keys/values to match on. Returns: boolean: True if contains all provided attributes, False otherwise. """ return all( getattr(self, key, None) == val for (key, val) in kwargs.items()) def metric_eval(self, feed_statuses: OrderedDict): """Select metric evaluation methon.""" return { METRIC_FRESH: self.freshness_eval, METRIC_RECENT_SYNC: self.freshness_eval, METRIC_STABLE: self.stability_eval }[self.metric_name](feed_statuses) def freshness_eval(self, statuses: OrderedDict) -> str: """Evaluate current status of monitor. Args: statuses (OrderedDict): ordered dict of date to status pairs. Order of insertion into dict should be most recent first. Returns: string: metric status constant METRIC_STATUS_xxx. """ def _ignore_feed_status(feed_due_time: str) -> bool: """Evaluate whether now time is before threshold_time. Args: feed_due_time (str): hours/minutes in 24hr formatted as '%H:%M' Returns: boolean: True if now < feed_due_time. """ hours, minutes = feed_due_time.split(':') delta = timedelta(hours=int(hours), minutes=int(minutes)) threshold_time = datetime( day=date.today().day, month=date.today().month, year=date.today().year) + delta ignore_status = True if datetime.now() > threshold_time: ignore_status = False return ignore_status health = METRIC_STATUS_OK feed_due_time = self.threshold_time expected_delay = self.threshold # Get the first day after threshold period # but first check if there is enought statuses statuses needed_statuses = set() for status_date in statuses: if ( date.fromisoformat(status_date) >= date.today() - timedelta(days=expected_delay) ): needed_statuses.add(statuses[status_date]) if self.success_status in needed_statuses: return METRIC_STATUS_OK if _ignore_feed_status(feed_due_time): return METRIC_STATUS_PENDING if self.success_status not in needed_statuses: return METRIC_STATUS_CRITICAL return health def stability_eval(self, statuses: OrderedDict) -> str: """Daily Stability health of monitor. Args: statuses (OrderedDict): ordered dict of date to status pairs. Order of insertion into dict should be most recent first. Returns: string: metric status constant METRIC_STATUS_xxx. """ health = METRIC_STATUS_OK period = config.STABILITY_MONITORING_PERIOD stability_threshold = 0 expected_delay = self.threshold feed = get_feed_by_id(self.feed_id) if not feed: return METRIC_STATUS_CRITICAL # Metric for weekly feeds elif feed.feed_type == ingestion_feed.INGESTION_TYPE_WEEKLY: period = config.WEEKLY_STABILITY_MONITORING_PERIOD period_statuses = list(statuses.values())[expected_delay:period] week = 7 # days for day in range(len(period_statuses)): if len(period_statuses[day:day+week]) < week: return health if self.success_status not in period_statuses[day:day+week]: return METRIC_STATUS_CRITICAL # Metric for daily feeds # cut threshold days from monitored period inspected_statuses = list(statuses.items())[expected_delay:period] # count unsuccessful statuses error_count = len([ status for _, status in inspected_statuses if status != self.success_status ]) # if unsuccessful statuses exceed the threshold # mark feed health ('stable' column) as 'critical' if error_count > stability_threshold: health = METRIC_STATUS_CRITICAL return health class FeedMonitorWithDayOfWeekPattern(FeedMonitor): """FeedMonitor class to calculate stability, considering active days .""" def __init__(self, feed_id: str, metric_name: str, threshold: int, threshold_time: Optional[str], success_status: str, active_days_of_week: [int]) -> None: """Initialize class with attributes. Args: feed_id: identifier for feed. metric_name: name of metric being monitored. threshold: recent days excluded from health metric. threshold_time: 24hr time of day in "%H:%M" format. Used to determine whether current day included in threshold. active_days_of_week: list of feed active days Monday is 0 and Sunday is 6 success_status: status which indicates successful ETL completion. """ self.active_days_of_week = active_days_of_week super().__init__(feed_id, metric_name, threshold, threshold_time, False, success_status) def stability_eval(self, statuses: OrderedDict) -> str: """Daily Stability health of monitor. Args: statuses (OrderedDict): ordered dict of date to status pairs. Order of insertion into dict should be most recent first. Returns: string: metric status constant METRIC_STATUS_xxx. """ statuses = {k: v for k, v in statuses.items() if datetime.strptime(k, '%Y-%m-%d').weekday() in self.active_days_of_week} return super().stability_eval(collections.OrderedDict(statuses)) FEED_MONITORS = [ FeedMonitor(feed_id='amazon_digital_services_theorchard', metric_name=METRIC_FRESH, threshold=4, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_digital_services_theorchard', metric_name=METRIC_STABLE, threshold=4, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_digital_services_sme', metric_name=METRIC_FRESH, threshold=4, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_digital_services_sme', metric_name=METRIC_STABLE, threshold=4, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_theorchard_streams', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_theorchard_streams', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_theorchard_sub_30_sec_streams', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_theorchard_sub_30_sec_streams', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_sme_streams', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_sme_streams', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_sme_sub_30_sec_streams', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_sme_sub_30_sec_streams', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_smejpintl_streams', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_smejpintl_streams', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_smejpintl_sub_30_sec_streams', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_smejpintl_sub_30_sec_streams', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # spotify charts FeedMonitor(feed_id='spotify_charts_daily_top_songs', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_charts_daily_top_songs', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_charts_weekly_top_songs', metric_name=METRIC_FRESH, threshold=2+7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_charts_weekly_top_songs', metric_name=METRIC_STABLE, threshold=2+7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='7', metric_name=METRIC_FRESH, threshold=9, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_unlimited', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_theorchard_unlimited', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_theorchard_unlimited', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_theorchard_prime', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_theorchard_prime', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_theorchard_adsupported', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_theorchard_adsupported', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_sme_unlimited', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_sme_unlimited', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_sme_prime', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_sme_prime', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_sme_adsupported', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='amazon_music_sme_adsupported', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amContentDemographics', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amContentDemographics', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amShazam', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amShazam', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amNonRoyaltySummaryStreams', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amNonRoyaltySummaryStreams', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amLibraryEvents', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amLibraryEvents', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amTotalLibraryAdds', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amTotalLibraryAdds', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amArtists', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amArtists', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amPlaylists', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amPlaylists', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amSongs', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amSongs', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amStreamsSummary', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amStreamsSummary', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amContainer', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_theorchard_amContainer', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amContentDemographics', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amContentDemographics', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amShazam', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amShazam', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amNonRoyaltySummaryStreams', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amNonRoyaltySummaryStreams', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amLibraryEvents', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amLibraryEvents', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amTotalLibraryAdds', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amArtists', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amArtists', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amPlaylists', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amPlaylists', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amSongs', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amSongs', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amStreamsSummary', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amStreamsSummary', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amContainer', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amContainer', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_sme_amTotalLibraryAdds', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amContentDemographics', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amContentDemographics', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amShazam', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amShazam', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amNonRoyaltySummaryStreams', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amNonRoyaltySummaryStreams', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amLibraryEvents', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amLibraryEvents', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amTotalLibraryAdds', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amTotalLibraryAdds', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amArtists', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amArtists', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amPlaylists', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amPlaylists', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amSongs', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amSongs', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amStreamsSummary', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amStreamsSummary', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amContainer', metric_name=METRIC_FRESH, threshold=2, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='apple_music_awal_amContainer', metric_name=METRIC_STABLE, threshold=2, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='deezer_daily_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='deezer_daily_sme', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='deezer_daily_theorchard', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='deezer_daily_sme', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='facebook', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='facebook', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='itunes_theorchard', metric_name=METRIC_FRESH, threshold=3, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='itunes_theorchard', metric_name=METRIC_STABLE, threshold=3, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='itunes_sme', metric_name=METRIC_FRESH, threshold=3, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='itunes_sme', metric_name=METRIC_STABLE, threshold=3, threshold_time='05:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='pandora_theorchard', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='pandora_theorchard', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='pandora_sme', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='pandora_sme', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='physical-warehouse-reports', metric_name=METRIC_FRESH, threshold=1, threshold_time='00:00', only_business_days=False, success_status=(ingestion_feed. INGESTION_STATUS_POPULATED_RAW_TABLE)), # Need to replace Youtube monthly feed # with more granular youtube monthly feeds # FeedMonitor(feed_id='youtube_monthly', # metric_name=METRIC_FRESH, # threshold=35, # threshold_time='00:00', # only_business_days=False, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # FeedMonitor(feed_id='youtube_monthly', # metric_name=METRIC_STABLE, # threshold=35, # threshold_time='00:00', # only_business_days=True, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_theorchard_aggregated_streams', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_theorchard_aggregated_streams', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_sme_aggregated_streams', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='spotify_sme_aggregated_streams', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='line_theorchard', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='line_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='line_smej', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='line_smej', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_theorchard', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_sme', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_sme', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_smejp', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_smejp', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_smejpintl', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_smejpintl', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='delphi_crm_complaint_tracking', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='delphi_crm_complaint_tracking', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='delphi_crm_held_emails', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='delphi_crm_held_emails', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='delphi_crm_mc_unsubscribe_data', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='delphi_crm_mc_unsubscribe_data', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='soundcloud_theorchard', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='soundcloud_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='soundcloud_sme', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='soundcloud_sme', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='awa_theorchard', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='awa_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='awa_smej', metric_name=METRIC_STABLE, threshold=2, threshold_time='00:00', only_business_days=False, success_status=( ingestion_feed.INGESTION_STATUS_POPULATED_RAW_TABLE)), FeedMonitor(feed_id='awa_smej', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=( ingestion_feed.INGESTION_STATUS_POPULATED_RAW_TABLE)), # FeedMonitor(feed_id='feature_fm_facebook', # metric_name=METRIC_FRESH, # threshold=2, # threshold_time='00:00', # only_business_days=False, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='vevo_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='vevo_sme', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='beatport', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='beatport', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='qq_theorchard', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='qq_theorchard', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='qq_sme', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='qq_sme', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kugou_theorchard', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kugou_theorchard', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kugou_sme', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kugou_sme', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kuwo_theorchard', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kuwo_theorchard', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kuwo_sme', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='kuwo_sme', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_fraudulent_streams_report', metric_name=METRIC_FRESH, threshold=80, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='snapchat_creations', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='snapchat_creations', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='snapchat_views', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='snapchat_views', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='gfk_physical', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='gfk_physical', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='gfk_streaming', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='gfk_streaming', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_production_theorchard', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_production_theorchard', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_production_sme', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_production_sme', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_consumption_theorchard', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_consumption_theorchard', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_consumption_sme', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='meta_daily_consumption_sme', metric_name=METRIC_STABLE, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # Cumulative reports FeedMonitor(feed_id='apple_id_mapping_sme', metric_name=METRIC_FRESH, threshold=3, threshold_time='05:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # FeedMonitor(feed_id='proper_daily_goodsin', # metric_name=METRIC_FRESH, # threshold=0, # threshold_time='00:00', # only_business_days=True, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # FeedMonitor(feed_id='proper_daily_sales', # metric_name=METRIC_FRESH, # threshold=0, # threshold_time='15:00', # only_business_days=True, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # FeedMonitor(feed_id='proper_daily_shortages', # metric_name=METRIC_FRESH, # threshold=0, # threshold_time='15:00', # only_business_days=True, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # FeedMonitor(feed_id='proper_daily_stock', # metric_name=METRIC_FRESH, # threshold=0, # threshold_time='00:00', # only_business_days=True, # success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='itunes_tickets', metric_name=METRIC_FRESH, threshold=0, threshold_time='13:30', # after 1st scheduled run 8 AM EST only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='itunes_hides', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_claim_theorchard', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_claim_sme', metric_name=METRIC_FRESH, threshold=4, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_channel_names_theorchard', metric_name=METRIC_FRESH, threshold=6, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_channel_names_sme', metric_name=METRIC_FRESH, threshold=6, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_neo4j', metric_name=METRIC_FRESH, threshold=6, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_neo4j', metric_name=METRIC_STABLE, threshold=6, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # Weekly reports FeedMonitor(feed_id='youtube_weekly', metric_name=METRIC_FRESH, threshold=16, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_weekly', metric_name=METRIC_STABLE, threshold=16, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='gfk', metric_name=METRIC_FRESH, threshold=11, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='gfk', metric_name=METRIC_STABLE, threshold=11, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_weekly', metric_name=METRIC_FRESH, threshold=11, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='tiktok_weekly', metric_name=METRIC_STABLE, threshold=11, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # Chartmetric FeedMonitor(feed_id='chartmetric_charts', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='chartmetric_charts', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='chartmetric_participants', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='chartmetric_tracks', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='chartmetric_socials', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='chartmetric_socials_backfill', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='chartmetric_playlists_share', metric_name=METRIC_FRESH, threshold=9, threshold_time='00:00', only_business_days=True, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # spotify_marque FeedMonitor( feed_id='spotify_marquee_theorchard_segment_level', metric_name=METRIC_FRESH, threshold=12, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_marquee_theorchard_campaign_level', metric_name=METRIC_FRESH, threshold=12, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_marquee_theorchard_segment_level', metric_name=METRIC_STABLE, threshold=12, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_marquee_theorchard_campaign_level', metric_name=METRIC_STABLE, threshold=12, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_track_metrics_playlist_atc', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_track_metrics_playlist_atc', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_track_metrics_track_passion', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_track_metrics_track_passion', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_track_metrics_track_conversion', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_track_metrics_track_conversion', metric_name=METRIC_STABLE, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='netease', metric_name=METRIC_FRESH, threshold=14, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='ticketek_event_booking', metric_name=METRIC_FRESH, threshold=1, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='seated_opt_ins', metric_name=METRIC_FRESH, threshold=3, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='seated_opt_ins', metric_name=METRIC_STABLE, threshold=3, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='bandsintown_artist_optins', metric_name=METRIC_FRESH, threshold=1, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='bandsintown_rsvp', metric_name=METRIC_FRESH, threshold=1, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='bandsintown_sony_optins', metric_name=METRIC_FRESH, threshold=1, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='bandsintown_ticketclick', metric_name=METRIC_FRESH, threshold=1, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='bandsintown_tracking', metric_name=METRIC_FRESH, threshold=1, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='physical_reporting_dpworld', metric_name=METRIC_FRESH, threshold=2, threshold_time='06:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), # YouTube Bulk Reports FeedMonitor(feed_id='youtube_asset_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_video_theorchard', metric_name=METRIC_FRESH, threshold=2, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_asset_conflict', metric_name=METRIC_FRESH, threshold=3, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_theorchard_asset', metric_name=METRIC_FRESH, threshold=5, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_theorchard_asset', metric_name=METRIC_STABLE, threshold=5, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_theorchard_video', metric_name=METRIC_FRESH, threshold=5, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_theorchard_video', metric_name=METRIC_STABLE, threshold=5, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor( feed_id='spotify_marketshare_week_ending', metric_name=METRIC_FRESH, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor( feed_id='spotify_marketshare_week_ending', metric_name=METRIC_STABLE, threshold=10, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # YouTube Bulk SME Reports FeedMonitor(feed_id='youtube_asset_sme', metric_name=METRIC_FRESH, threshold=7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_video_sme', metric_name=METRIC_FRESH, threshold=7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_sme_asset', metric_name=METRIC_FRESH, threshold=7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_sme_asset', metric_name=METRIC_STABLE, threshold=7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_sme_video', metric_name=METRIC_FRESH, threshold=7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor(feed_id='youtube_facts_sme_video', metric_name=METRIC_STABLE, threshold=7, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), # Outgoing feeds, FeedMonitor(feed_id='proper_new_releases_tracks', metric_name=METRIC_FRESH, threshold=0, threshold_time='09:45', only_business_days=False, success_status=outgoing_feed.OUTGOING_STATUS_SENT), FeedMonitor(feed_id='proper_changed_releases', metric_name=METRIC_FRESH, threshold=0, threshold_time='09:45', only_business_days=False, success_status=outgoing_feed.OUTGOING_STATUS_SENT), # ActivityDetector feeds FeedMonitor( feed_id='chartmetric_spike_detector', metric_name=METRIC_FRESH, threshold=1, threshold_time='00:00', only_business_days=False, success_status=ad_feed.ACTIVITY_DETECTOR_STATUS_PROCESSED, ), # Marketshare feeds FeedMonitor( feed_id='amazon_prime_marketshare', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='amazon_unlimited_marketshare', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='deezer_marketshare', metric_name=METRIC_FRESH, threshold=105, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='itunes_marketshare', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_marketshare', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='youtube_red_marketshare', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), # Monthly feeds FeedMonitor( feed_id='tiktok_openescrow', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='insta_reels_creations', metric_name=METRIC_FRESH, threshold=95, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='insta_reels_views', metric_name=METRIC_FRESH, threshold=95, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='spotify_artificial_streams', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), FeedMonitor( feed_id='apple_financial', metric_name=METRIC_FRESH, threshold=75, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), ] for report in flows.YOUTUBE_MONTHLY_REPORTS: FEED_MONITORS.append( FeedMonitor( # files usually appears 10-13 days after the month ends feed_id=f'youtube_monthly_{report}', metric_name=METRIC_FRESH, threshold=30 * 2 + 15, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ), ) for report in flows.YouTubeBulkReports.REPORTS_THEORCHARD: feed_id = f'youtube_bulk_reports_theorchard_{report}' monitors = [ FeedMonitor( feed_id=feed_id, metric_name=METRIC_FRESH, threshold=5, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), FeedMonitor( feed_id=feed_id, metric_name=METRIC_STABLE, threshold=5, threshold_time='00:00', only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED), ] FEED_MONITORS.extend(monitors) for report in flows.YouTubeBulkReports.REPORTS_SME: feed_id = f'youtube_bulk_reports_sme_{report}' if report in flows.YouTubeBulkReports.SME_ONLY_DOWNLOAD_REPORTS: success_status = ingestion_feed.INGESTION_STATUS_DOWNLOADED else: success_status = ingestion_feed.INGESTION_STATUS_INGESTED threshold = 7 monitors = [ FeedMonitor( feed_id=feed_id, metric_name=METRIC_FRESH, threshold=threshold, threshold_time='00:00', only_business_days=False, success_status=success_status), FeedMonitor( feed_id=feed_id, metric_name=METRIC_STABLE, threshold=threshold, threshold_time='00:00', only_business_days=False, success_status=success_status), ] if report in flows.YouTubeBulkReports.SME_NO_STABLE_METRIC_REPORTS: monitors.pop(1) FEED_MONITORS.extend(monitors) for feed_id in flows.Tadas.FEEDS: monitor = FeedMonitor( feed_id=feed_id, metric_name=METRIC_FRESH, threshold=flows.Tadas.FRESH_THRESHOLD, threshold_time=flows.Tadas.FRESH_THRESHOLD_TIME, only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED, ) FEED_MONITORS.append(monitor) for feed_id in flows.AmazonDatapulse.FEEDS: monitor = FeedMonitor( feed_id=feed_id, metric_name=METRIC_FRESH, threshold=flows.AmazonDatapulse.FRESH_THRESHOLD, threshold_time=flows.AmazonDatapulse.FRESH_THRESHOLD_TIME, only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED) FEED_MONITORS.append(monitor) monitor = FeedMonitor( feed_id=feed_id, metric_name=METRIC_STABLE, threshold=flows.AmazonDatapulse.FRESH_THRESHOLD, threshold_time=flows.AmazonDatapulse.FRESH_THRESHOLD_TIME, only_business_days=False, success_status=ingestion_feed.INGESTION_STATUS_INGESTED) FEED_MONITORS.append(monitor) # delphi for line in delphi_feed.get_delphi_feeds_config(): feed_id = (f'delphi-{line["data_source_name"]}-' f'{line["licensor_name"]}-{line["report_name"]}') is_fresh = line['fresh_metric'] if is_fresh == '1': refresh_monitor = FeedMonitor( feed_id=feed_id, metric_name=METRIC_FRESH, threshold=int(line['data_gap']), threshold_time='00:00', only_business_days=bool(line['only_business_days']), success_status=delphi_feed.INGESTION_STATUS_INGESTED) FEED_MONITORS.append(refresh_monitor) is_stable = line['stable_metric'] if is_stable == '1': if line['data_source_name'] == 'tiktokreporting' and \ line['report_name'] == 'report_meta_song': stable_monitor = FeedMonitorWithDayOfWeekPattern( feed_id=feed_id, metric_name=METRIC_STABLE, threshold=int(line['data_gap']), threshold_time='00:00', active_days_of_week=[0, 1, 2, 3, 6], success_status=delphi_feed.INGESTION_STATUS_INGESTED) else: stable_monitor = FeedMonitor( feed_id=feed_id, metric_name=METRIC_STABLE, threshold=int(line['data_gap']), threshold_time='00:00', only_business_days=bool(line['only_business_days']), success_status=delphi_feed.INGESTION_STATUS_INGESTED) FEED_MONITORS.append(stable_monitor) def get_feed_by_id( feed_id: Union[int, str], ingestion_only: bool = False ) -> Optional[ Union[ IngestionFeed, OutgoingFeed, ActivityDetectorFeed, DelphiFeed ] ]: """Retrieve instance of IngestionFeed by id. Args: Args: feed_id (str): feed id as a string. Returns: IngestionFeed - instance with feed id, or None if not found. """ active_feeds = ingestion_feed.active_feeds if not ingestion_only: active_feeds = [ *ingestion_feed.active_feeds, *ingestion_feed.active_youtube_bulk_feeds, *ingestion_feed.active_youtube_bulk_sme_feeds, *ingestion_feed.active_marketshare_feeds, *outgoing_feed.active_feeds, *ad_feed.active_feeds, *delphi_feed.active_feeds ] for feed in active_feeds: if feed.match(feed_id=feed_id): return feed def find_monitors(**kwargs) -> List[FeedMonitor]: """Find list of FeedMonitors that match kwargs. Args: kwargs: keys/values to filter by. Returns: list: all found FeedMonitors. """ return list(monitor for monitor in FEED_MONITORS if monitor.match(**kwargs)) def get_metric_status( feed: Optional[ Union[ IngestionFeed, OutgoingFeed, ActivityDetectorFeed, DelphiFeed ] ], feed_statuses: OrderedDict, ) -> List[Dict]: """Evaluate FeedMonitor status configured for feed given prior statuses. Args: feed (IngestionFeed): IngestionFeed instance. feed_statuses (dict): map of dates to feed status. Returns: list: for all configured FeedMonitors for this feed, a map of metric status attributes. """ metrics = [] monitors = find_monitors(feed_id=str(feed.feed_id)) for monitor in monitors: status = monitor.metric_eval(feed_statuses) # emit json metrics atrributes to be auto-parsed by datadog logs print(json.dumps({ 'feed_name': getattr(feed, 'feed_name', None) or str(feed.feed_id), 'feed_id': str(feed.feed_id), 'status': status, 'metrics_category': _METRIC_CATEGORY[monitor.metric_name], 'threshold': monitor.threshold, }), flush=True) metrics.append({ 'name': monitor.metric_name, 'status': status, 'threshold': monitor.threshold, 'only_business_days': monitor.only_business_days }) return metrics