# pylint: disable=protected-access,unused-argument,too-many-arguments import json from contextlib import nullcontext as does_not_raise from http import HTTPStatus from os.path import join as p_join from typing import Any, Callable, Union from unittest.mock import MagicMock, Mock, patch import awswrangler as wr import pandas as pd import pytest from _pytest.python_api import RaisesContext from requests import Response from slz_spotify_charts_scraper.const import SpotifyChartRecurrenceType, SpotifyChartType from slz_spotify_charts_scraper.exceptions import ChartsDataParsingError from slz_spotify_charts_scraper.service import SpotifyChartsService from tests import FIXTURES_PATH from tests.entities import BucketFilesCounter, BucketPathsTuple def fail_n_times(times_to_fail: int, exception_to_raise: Exception, return_success: Any): counter = 0 def counting_effect(*args, **kwargs): nonlocal counter if counter < times_to_fail: counter += 1 raise exception_to_raise return return_success return counting_effect @pytest.mark.parametrize( # pylint: disable=invalid-string-quote 'side_effect, raises, expected_attempts', [ ( fail_n_times(0, AttributeError('Expected error message'), 'token123'), does_not_raise(), 1, ), ( fail_n_times(1, AttributeError("'Response' object has no attribute 'txt'"), 'token123'), does_not_raise(), 2, ), ( fail_n_times(2, AttributeError("'Response' object has no attribute 'txt'"), 'token123'), does_not_raise(), 3, ), # Retry for only 3 times, then fail. ( fail_n_times(3, AttributeError("'Response' object has no attribute 'txt'"), 'token123'), pytest.raises(AttributeError), 3, ), # Unexpected exception message, should be no retries. ( fail_n_times(2, AttributeError('Unexpected error'), 'token123'), pytest.raises(AttributeError), 1 ), # Another unexpected error, should be raised immediately ( fail_n_times(2, ValueError('Another unexpected error'), 'token123'), pytest.raises(ValueError), 1 ), ] ) # yapf: disable @patch('slz_spotify_charts_scraper.service.requests.get') def test__fetch_chart_data__retry( requests_mock: MagicMock, spotify_service_stateless: SpotifyChartsService, side_effect: Callable, raises: Union[does_not_raise, RaisesContext], expected_attempts: int ): # yapf: disable spotify_service_stateless. \ _credentials_client. \ get_access_token.side_effect = side_effect dummy_url = 'dummy_url' expected = b'dummy_context' custom_response = Response() custom_response.status_code = HTTPStatus.OK custom_response._content = expected requests_mock.return_value = custom_response with raises: result = spotify_service_stateless._fetch_chart_data(dummy_url) assert result == expected requests_mock.assert_called_once_with( dummy_url, headers={'Authorization': 'Bearer token123'} ) assert spotify_service_stateless.\ _credentials_client.\ get_access_token.call_count == expected_attempts @pytest.mark.parametrize( 'chart_raw_data_path,expected_path', [ ( p_join(FIXTURES_PATH, 'data', 'top200_daily_us_raw.json'), p_join(FIXTURES_PATH, 'data', 'top200_daily_us.json'), ) ] ) @pytest.mark.freeze_time('2021-10-11T10:51:08') def test_normalize_charts_data( chart_raw_data_path: str, expected_path: str, spotify_service_stateless: SpotifyChartsService, ): with open(chart_raw_data_path, 'r', encoding='utf-8') as file_: data = json.load(file_) parsed_data = spotify_service_stateless.normalize_charts_data(data, 'us') with open(expected_path, 'r', encoding='utf-8') as file_: expected_data = json.load(file_) assert parsed_data == expected_data @pytest.mark.parametrize( 'chart_raw_data_path,expected', [( p_join(FIXTURES_PATH, 'data', 'top200_daily_us.html'), '2da691953f64c43a4f9903bedd34b647', )] ) @pytest.mark.freeze_time('2021-10-21') @pytest.mark.skip( reason='hashing of a parquet file gives unpredictable results. Not reproducible locally' ) def test_chart_raw_data_path_parquet_hash(chart_raw_data_path, expected, spotify_service_stateless): with open(chart_raw_data_path, 'r', encoding='utf-8') as file_: data = file_.read() parsed_data = spotify_service_stateless.parse_spotify_charts_data(data, 'us') data_frame = pd.DataFrame(parsed_data) file_hash = spotify_service_stateless.calc_file_parquet_hash( data_frame, columns_to_drop=['timestamp'] ) assert expected == file_hash @pytest.mark.parametrize( 'chart_raw_data_path,expected', [ ( p_join(FIXTURES_PATH, 'data', 'top200_daily_us_raw.json'), '33c4e168f0a72080a31e1211c5f0098b', ) ] ) @pytest.mark.freeze_time('2021-10-21') def test_chart_raw_data_path_json_hash(chart_raw_data_path, expected, spotify_service_stateless): with open(chart_raw_data_path, 'r', encoding='utf-8') as file_: data = json.load(file_) normalized_data = spotify_service_stateless.normalize_charts_data(data, 'us') file_hash = spotify_service_stateless.calc_file_json_hash( normalized_data, columns_to_drop=['timestamp'] ) assert expected == file_hash @pytest.mark.parametrize( 'chart_type, recurrence_type, expected', [ ( SpotifyChartType.REGIONAL, SpotifyChartRecurrenceType.DAILY, 'https://example.com/spotifycharts/regional-us-daily/2021/08/01', ), ( SpotifyChartType.VIRAL, SpotifyChartRecurrenceType.DAILY, 'https://example.com/spotifycharts/viral-us-daily/2021/08/01', ), ( SpotifyChartType.REGIONAL, SpotifyChartRecurrenceType.WEEKLY, 'https://example.com/spotifycharts/regional-us-weekly/2021/08/01', ), ( SpotifyChartType.VIRAL, SpotifyChartRecurrenceType.WEEKLY, 'https://example.com/spotifycharts/viral-us-weekly/2021/08/01', ), ] ) def test__get_chart_url( chart_type: SpotifyChartType, recurrence_type: SpotifyChartRecurrenceType, expected: str, spotify_service_stateless: SpotifyChartsService, ): spotify_service_stateless._payload.chart_type = chart_type spotify_service_stateless._payload.recurrence_type = recurrence_type spotify_service_stateless._config.spotify_charts_url_base = 'https://example.com/' result = spotify_service_stateless._get_chart_url('us') assert result == expected @pytest.mark.parametrize( 'given, expected, raises', [ ('spotify:track:02MWAaffLxlfxAUY7c5dvx', '02MWAaffLxlfxAUY7c5dvx', does_not_raise()), ('spotify:track:02MWAaffLxlfxAUY7c5dv', '', pytest.raises(ChartsDataParsingError)), ('spotify:track:1', '', pytest.raises(ChartsDataParsingError)), ('spotify:track:abc', '', pytest.raises(ChartsDataParsingError)), ('spotify:track:abc123', '', pytest.raises(ChartsDataParsingError)), ('spotify:track:?.-abc123', '', pytest.raises(ChartsDataParsingError)), ('track:spotify:1', '', pytest.raises(ChartsDataParsingError)), (':1', '', pytest.raises(ChartsDataParsingError)), ('', '', pytest.raises(ChartsDataParsingError)), ] ) def test__get_track_id_from_uri( given: str, expected: str, raises: Union[RaisesContext, does_not_raise], spotify_service_stateless: SpotifyChartsService, ): with raises: result = spotify_service_stateless._get_track_id_from_uri(given) assert result == expected @pytest.mark.parametrize( 'status_code, is_hash_changed, is_valid, expected_counters', [ (HTTPStatus.OK, True, True, BucketFilesCounter(0, 0, 1)), (HTTPStatus.OK, True, False, BucketFilesCounter(0, 2, 0)), (HTTPStatus.OK, False, True, BucketFilesCounter(0, 0, 0)), (HTTPStatus.OK, False, False, BucketFilesCounter(0, 0, 0)), (HTTPStatus.NOT_FOUND, True, True, BucketFilesCounter(0, 0, 0)), (HTTPStatus.FORBIDDEN, True, True, BucketFilesCounter(0, 0, 0)), ] ) @patch('slz_spotify_charts_scraper.service.requests.get') def test__process_market_charts_data__stateless( requests_mock: MagicMock, s3_client, spotify_service_stateless: SpotifyChartsService, charts_page_content: bytes, status_code: HTTPStatus, bucket_paths_stateless: BucketPathsTuple, is_hash_changed: bool, is_valid: bool, expected_counters: BucketFilesCounter, ): # Use shorter fixture names to improve readability spotify_service = spotify_service_stateless bucket_paths = bucket_paths_stateless custom_response = Response() custom_response.status_code = status_code custom_response._content = charts_page_content requests_mock.return_value = custom_response spotify_service._snapshot_repository.is_hash_updated = Mock(return_value=is_hash_changed) spotify_service._validator.validate_data_frame = Mock(return_value=is_valid) spotify_service._process_market_charts_data(market='us') assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == expected_counters.quarantine assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == expected_counters.corrupted assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == expected_counters.decompressed