import datetime import logging from tadas.monitoring import sensors from tadas.platform import config from tadas.platform import context as contexts from tadas.platform import metrics as metrics_utils from tadas.snowflake.facts_source import SUPPORTED_DSPS, load_available_dates logger = logging.getLogger(__name__) FRESHNESS_THRESHOLD_PER_DSP = { # Spotify avg delivery at 15:00 UTC # threshold aligned with DBT freshness monitor 19:45 UTC 'spotify': datetime.timedelta(days=2, hours=19, minutes=45), # Apple Music avg delivery at 19:00 UTC # threshold aligned with DBT freshness monitor 19:45 UTC 'apple': datetime.timedelta(days=2, hours=19, minutes=45), # tiktok is expected 1.5 hours earlier than TADAS's FRESHNESS_THRESHOLD_HOURS 'tiktok': datetime.timedelta(hours=74 - 1.5), 'amazon': datetime.timedelta(days=2, hours=24), 'youtube': datetime.timedelta(days=2, hours=24), } def get_threshold(source: str): for dsp, threshold in FRESHNESS_THRESHOLD_PER_DSP.items(): if source.startswith(dsp): return threshold return None def run(): available_dates = load_available_dates( dsps=SUPPORTED_DSPS, max_days_back=config.get('DAYS_BACK'), ) for source, dates in available_dates.items(): if not dates: logger.warning(f'No available dates for {source}') continue most_recent = max(dates) metrics = metrics_utils.Metrics( category='data_availability', context=contexts.load_context(), ) metrics.add_metric('data_source', source) metrics.add_metric('max_available_date', most_recent.isoformat()) metrics.send() threshold = get_threshold(source) if threshold: sensors.run_freshness_check(name=source, report_date=most_recent, threshold=threshold) else: logger.warning(f'No threshold set for {source}, skipping freshness check')