"""Unit tests for data availability model layer.""" import datetime import gzip from unittest.mock import patch import pytest from analytics.connectors import redis from analytics.utils.data_availability import ( FIELD_FEED_ID, FIELD_SKIPS_SAVES, FIELD_STORE_ID, FIELD_STREAMS, FIELD_TYPES, get_downloads_max_available_date, get_max_available_date, get_outage_stores, get_videos_max_available_date, ) MOCK_SQL = "{subaccount_clause}" @pytest.fixture def mock_outage_snowflake_fetchall(): """Mock Store Outages Snowflake fetchall.""" with patch("analytics.utils.data_availability.snowflake.fetchall") as fetchall: fetchall.return_value = [ (1, 4, True, True), (2, 3, False, True), (3, 2, True, False), (4, 1, False, False), ] yield fetchall @pytest.fixture def mock_max_available_date_snowflake_fetchall(): """Mock Store Outages Snowflake fetchall.""" with patch("analytics.utils.data_availability.snowflake.fetchall") as fetchall: fetchall.return_value = [(datetime.date(2018, 7, 6),)] yield fetchall @pytest.fixture def mock_max_available_date_cache(): """Mock Store Outages Snowflake fetchall.""" with patch("analytics.connectors.redis.client.get") as cache: cache.return_value = gzip.compress(b'"2018-07-06"') yield cache @pytest.fixture def mock_downloads_max_available_date_cache(): """Mock Store Outages Snowflake fetchall.""" with patch("analytics.connectors.redis.client.get") as cache: cache.return_value = gzip.compress(b'"2018-07-07"') yield cache @pytest.fixture def mock_videos_max_available_date_cache(): """Mock Store Outages Snowflake fetchall.""" with patch("analytics.connectors.redis.client.get") as cache: cache.return_value = gzip.compress(b'"2018-07-06"') yield cache def test_get_outage_stores(mock_outage_snowflake_fetchall): """Test get_outage_stores function.""" redis.client.flushall() # flush Fakeredis cache result = get_outage_stores({"feed_ids": [1, 4]}) assert result == [ { FIELD_FEED_ID: 1, FIELD_STORE_ID: 4, FIELD_TYPES: [FIELD_STREAMS, FIELD_SKIPS_SAVES], }, {FIELD_FEED_ID: 2, FIELD_STORE_ID: 3, FIELD_TYPES: [FIELD_SKIPS_SAVES]}, {FIELD_FEED_ID: 3, FIELD_STORE_ID: 2, FIELD_TYPES: [FIELD_STREAMS]}, ] assert mock_outage_snowflake_fetchall.call_count == 1 def test_get_max_available_date( mock_max_available_date_cache, mock_max_available_date_snowflake_fetchall ): """Test get_max_available_date function.""" result = get_max_available_date() assert result == "2018-07-06" def test_get_max_available_date_flag_off_reads_live_table( mock_max_available_date_snowflake_fetchall, ): """Flag off (autouse default): loads the original store_high_water_mark query.""" redis.client.flushall() # force a cache miss so the query file is loaded result = get_max_available_date({7: {"name": "feed"}}) assert result == "2018-07-06" sql = mock_max_available_date_snowflake_fetchall.call_args[0][0] assert "data_availability_by_store_daily_published" not in sql assert "data_availability_by_store_daily" in sql def test_get_max_available_date_flag_on_reads_published_snapshot( mock_max_available_date_snowflake_fetchall, ): """insights_published_max_available_date on: loads the _PUBLISHED query.""" redis.client.flushall() with patch( "analytics.utils.data_availability." "is_insights_published_max_available_date_enabled", return_value=True, ): result = get_max_available_date({7: {"name": "feed"}}) assert result == "2018-07-06" sql = mock_max_available_date_snowflake_fetchall.call_args[0][0] assert "data_availability_by_store_daily_published" in sql def test_get_downloads_max_available_date( mock_downloads_max_available_date_cache, mock_max_available_date_snowflake_fetchall ): """Test get_downloads_max_available_date function.""" result = get_downloads_max_available_date() assert result == "2018-07-07" def test_get_videos_max_available_date( mock_videos_max_available_date_cache, mock_max_available_date_snowflake_fetchall, ): """Test get_max_available_date function.""" result = get_videos_max_available_date() assert result == "2018-07-06" @pytest.fixture def mock_get_feed_outages_v2_snowflake_fetchall(): """Mock get_feed_outages_v2 Snowflake fetchall.""" with patch("analytics.utils.data_availability.snowflake.fetchall") as fetchall: fetchall.return_value = [ ( "sme", # distributor "SME Feed", # feed_name "1", # feed_id "4", # store_id "2023-04-15", # store_high_water_mark "2023-04-14", # streaming_stores_high_watermark "[]", # missing_dates_before_store_high_watermark True, # has_streams_outage ), ( "theorchard", "Orchard Feed", "2", "3", "2023-04-10", "2023-04-12", '["2023-04-09","2023-04-08"]', False, ), ] yield fetchall @pytest.fixture def mock_datetime_now(): """Mock datetime.now to return a fixed date.""" mock_date = datetime.datetime(2023, 4, 20, tzinfo=datetime.timezone.utc) with patch("analytics.utils.data_availability.datetime") as mock_dt: mock_dt.now.return_value = mock_date mock_dt.strptime.side_effect = datetime.datetime.strptime mock_dt.timezone = datetime.timezone mock_dt.timedelta = datetime.timedelta yield mock_dt def test_calculate_update_status(mock_datetime_now): """Test _calculate_update_status function.""" from analytics.utils.data_availability import _calculate_update_status available_feeds = {1: {"update_threshold_days": 7}, 2: {"update_threshold_days": 5}} # Test case 1: Within threshold (today - high_water_mark <= threshold) result1 = _calculate_update_status("2023-04-15", 1, available_feeds) assert result1 == "UPDATED" # Test case 2: Outside threshold (today - high_water_mark > threshold) result2 = _calculate_update_status("2023-04-10", 2, available_feeds) assert result2 == "WAITING_FOR_UPDATES" def test_get_feed_outages_v2( mock_get_feed_outages_v2_snowflake_fetchall, mock_datetime_now ): """Test get_feed_outages_v2 function.""" from analytics.utils.data_availability import get_feed_outages_v2 redis.client.flushall() # flush Fakeredis cache available_feeds = {1: {"update_threshold_days": 7}, 2: {"update_threshold_days": 5}} result = get_feed_outages_v2(available_feeds) assert len(result) == 2 # Check first feed record assert result[0]["distributor"] == "sme" assert result[0]["feed_id"] == 1 assert result[0]["store_id"] == 4 assert result[0]["store_high_water_mark"] == "2023-04-15" assert result[0]["update_status"] == "UPDATED" assert result[0]["streaming_stores_high_watermark"] == "2023-04-14" assert result[0]["missing_dates_before_store_high_watermark"] == [] assert result[0]["has_streams_outage"] is True # Check second feed record assert result[1]["distributor"] == "theorchard" assert result[1]["feed_id"] == 2 assert result[1]["store_id"] == 3 assert result[1]["store_high_water_mark"] == "2023-04-10" assert result[1]["update_status"] == "WAITING_FOR_UPDATES" assert result[1]["streaming_stores_high_watermark"] == "2023-04-12" assert result[1]["missing_dates_before_store_high_watermark"] == [ "2023-04-08", "2023-04-09", ] assert result[1]["has_streams_outage"] is False assert mock_get_feed_outages_v2_snowflake_fetchall.call_count == 1 @pytest.fixture def mock_feed_outages_snowflake_fetchall(): """Mock store_outages fetchall: (feed_id, store_id, has_streams, has_skips).""" with patch("analytics.utils.data_availability.snowflake.fetchall") as fetchall: fetchall.return_value = [ (1, 286, True, False), (3, 708, False, True), ] yield fetchall def test_get_feeds_with_outages_survives_cache_roundtrip( mock_feed_outages_snowflake_fetchall, ): """Outage error blocks must survive the redis cache round-trip. get_feed_outages is cached via cache_in_redis, which serializes through JSON -- coercing its integer feed-id keys to strings. The cache-hit lookup must still match, otherwise every request after the first silently drops every outage error block (the cached result looks empty). """ redis.client.flushall() # flush fakeredis cache from analytics.utils.data_availability import get_feeds_with_outages first = get_feeds_with_outages() # cache miss -> populates the cache second = get_feeds_with_outages() # cache hit -> JSON round-trip # the cached (string-keyed) result must behave identically to the fresh one assert first == second # feeds 1 and 3 are in outage and must carry an error block on both calls for feeds in (first, second): assert feeds[1]["error"] == {"types": [FIELD_STREAMS], "code": "unreliable"} assert feeds[3]["error"] == { "types": [FIELD_SKIPS_SAVES], "code": "unreliable", } # snowflake hit once; the second call was served from cache assert mock_feed_outages_snowflake_fetchall.call_count == 1 def test_get_feeds_with_outages_v2_watermark_structure(): """Test get_feeds_with_outages_v2 returns correct high_water_marks structure. This test verifies: - high_water_marks array contains all three distributors (sme, theorchard, awal) - Watermarks are extracted deterministically (one per distributor) - Each watermark has correct structure with distributor, type, and high_water_mark fields """ from analytics.utils.data_availability import get_feeds_with_outages_v2 # Mock data simulating multiple feeds per distributor with same watermark mock_feed_data = [ { "distributor": "sme", "feed_name": "Spotify", "feed_id": 1, "store_id": 286, "store_high_water_mark": "2023-04-15", "streaming_stores_high_watermark": "2023-04-14", # SME watermark "missing_dates_before_store_high_watermark": [], "has_streams_outage": False, }, { "distributor": "sme", "feed_name": "Apple Music", "feed_id": 3, "store_id": 286, "store_high_water_mark": "2023-04-15", "streaming_stores_high_watermark": "2023-04-14", # Same SME watermark "missing_dates_before_store_high_watermark": [], "has_streams_outage": False, }, { "distributor": "theorchard", "feed_name": "Spotify", "feed_id": 1, "store_id": 286, "store_high_water_mark": "2023-04-16", "streaming_stores_high_watermark": "2023-04-15", # Orchard watermark "missing_dates_before_store_high_watermark": [], "has_streams_outage": False, }, { "distributor": "theorchard", "feed_name": "Apple Music", "feed_id": 3, "store_id": 286, "store_high_water_mark": "2023-04-16", "streaming_stores_high_watermark": "2023-04-15", # Same Orchard watermark "missing_dates_before_store_high_watermark": [], "has_streams_outage": False, }, { "distributor": "awal", "feed_name": "Spotify", "feed_id": 1, "store_id": 286, "store_high_water_mark": "2023-04-13", "streaming_stores_high_watermark": "2023-04-12", # AWAL watermark "missing_dates_before_store_high_watermark": [], "has_streams_outage": False, }, { "distributor": "awal", "feed_name": "Apple Music", "feed_id": 3, "store_id": 286, "store_high_water_mark": "2023-04-13", "streaming_stores_high_watermark": "2023-04-12", # Same AWAL watermark "missing_dates_before_store_high_watermark": [], "has_streams_outage": False, }, ] with patch( "analytics.utils.data_availability.get_available_feeds" ) as mock_available_feeds, patch( "analytics.utils.data_availability.get_feed_outages_v2" ) as mock_get_feed_outages: mock_available_feeds.return_value = {1: {}, 3: {}} mock_get_feed_outages.return_value = mock_feed_data result = get_feeds_with_outages_v2() # Verify response structure assert "feed_statuses" in result assert "high_water_marks" in result # Verify high_water_marks array high_water_marks = result["high_water_marks"] assert len(high_water_marks) == 3 # Verify all three distributors are present distributors = {hwm["distributor"] for hwm in high_water_marks} assert distributors == {"sme", "theorchard", "awal"} # Verify each watermark has correct structure and values watermark_map = {hwm["distributor"]: hwm for hwm in high_water_marks} assert watermark_map["sme"]["type"] == "STREAMING_STORES" assert watermark_map["sme"]["high_water_mark"] == "2023-04-14" assert watermark_map["theorchard"]["type"] == "STREAMING_STORES" assert watermark_map["theorchard"]["high_water_mark"] == "2023-04-15" assert watermark_map["awal"]["type"] == "STREAMING_STORES" assert watermark_map["awal"]["high_water_mark"] == "2023-04-12" # Verify feed_statuses contains all 6 feeds assert len(result["feed_statuses"]) == 6