import boto3 import json from datetime import datetime from moto import mock_s3 from requests_mock import Mocker from typing import Iterable, List from download_spotify_charts import config, index from download_spotify_charts.constants import CSV_HEADER, CSV_NOTE, ChartBreakdown, ChartType from download_spotify_charts.logger import get_logger delphi_country_codes = ("us", "worldwide", "gb", "ca", "de", "cu", "it") local_country_codes = ("us", "global", "gb", "ca", "de", "cu", "it") def get_charts() -> dict: return { "items": [ { "chart_id": f"{ChartType.values()[i % 2]}_{ChartBreakdown.values()[i % 2]}_{delphi_country_codes[i]}", "country_code": delphi_country_codes[i], "type": ChartType.values()[i % 2], "breakdown": ChartBreakdown.values()[i % 2], } for i in range(4) ] } def get_dates() -> dict: return { "breakdown_chart_type_country_code": { c_breakdown: { c_type: { delphi_country_codes[i]: { "max_date": f"2022-04-1{4 if c_breakdown == ChartBreakdown.WEEKLY.value else i % 2 + 4}" } for i in range(5) } for c_type in ChartType.values() } for c_breakdown in ChartBreakdown.values() } } def get_tracks() -> dict: return { "items": [ { "metrics": {"position": i, "date_streams": 100 * i + i}, "public_meta": { "name": f"track_{i}", "artists": [{"name": f"artist_{i}_{j}"} for j in range(1, i)], "uri": f"uri_{i}", }, } for i in range(1, 5) ] } def generate_row(data: Iterable) -> str: return ",".join(f"\"{item}\"" for item in data) + "\n" def get_csv(chart_type: ChartType) -> str: result = "" if chart_type == ChartType.REGIONAL and config.PUT_NOTE_TO_CSV: result += generate_row(CSV_NOTE) if config.PUT_HEADER_TO_CSV: result += generate_row(CSV_HEADER[chart_type]) for i in range(1, 5): data = [str(i), f"track_{i}", ", ".join(f"artist_{i}_{j}" for j in range(1, i))] if chart_type == ChartType.REGIONAL: data.append(str(100 * i + i)) data.append(f"uri_{i}") result += generate_row(data) return result def get_config(data: List[int] = None) -> dict: if not data: data = [0] * 6 return { local_country_codes[i]: { "last_date": f"2022-04-{(14 + step):02d}", "ts": f"2022-04-{(14 + step):02d}T21:15:28.762612", } for i, step in enumerate(data) } def parse_config(data: dict) -> dict: return {cc: dates["last_date"] for cc, dates in data.items()} @mock_s3 @Mocker(kw="requests_mocker") def test_handler(mocker, **kwargs): fake_datetime = datetime(2022, 4, 15, 14, 5, 11) mocked_index_datetime = mocker.patch("download_spotify_charts.index.datetime") mocked_index_datetime.utcnow.return_value = fake_datetime mocked_index_datetime.fromisoformat.side_effect = datetime.fromisoformat mocked_utils_datetime = mocker.patch("download_spotify_charts.utils.datetime") mocked_utils_datetime.utcnow.return_value = fake_datetime requests_mocker = kwargs["requests_mocker"] s3_resource = boto3.resource("s3", region_name=config.AWS_DEFAULT_REGION) s3_resource.create_bucket(Bucket=config.AWS_S3_BUCKET_NAME) bucket = s3_resource.Bucket(config.AWS_S3_BUCKET_NAME) base_path = "spotifycharts/config" for c_type in ChartType.values(): for c_breakdown in ChartBreakdown.values(): bucket.put_object( Key=f"{base_path}/{c_breakdown}-{c_type}.json", Body=json.dumps( get_config( [-6, 0] if c_breakdown == ChartBreakdown.WEEKLY.value and c_type == ChartType.REGIONAL.value else None ) ).encode(), ContentType="text/json", ) requests_mocker.register_uri("GET", f"{config.DSP_URI}/delphi/spotify/charts", json=get_charts()) requests_mocker.register_uri("GET", f"{config.DSP_URI}/delphi/spotify/charts/data-health/status", json=get_dates()) requests_mocker.register_uri("GET", f"{config.DSP_URI}/delphi/spotify/charts/tracks", json=get_tracks()) index.handler(get_logger(1), {}) s3_files = list(bucket.objects.all()) assert len(s3_files) == 10 for file in s3_files: file_data = s3_resource.Object(config.AWS_S3_BUCKET_NAME, file.key).get()["Body"].read().decode() if ".json" in file.key: c_breakdown, c_type = file.key.split("/")[-1].split(".")[0].split("-") if c_breakdown == ChartBreakdown.WEEKLY.value and c_type == ChartType.REGIONAL.value: assert parse_config(json.loads(file_data)) == parse_config(get_config([0] * 3)) elif c_breakdown == ChartBreakdown.DAILY.value and c_type == ChartType.VIRAL.value: assert parse_config(json.loads(file_data)) == parse_config(get_config([0, 1, 0, 1, 0, 0])) else: assert json.loads(file_data) == get_config() else: c_type = ChartType(file.key.split("/")[2]) assert file_data == get_csv(c_type)