"""Test stateless ApplicationManager class methods. From technical point of view tests in this module are of integration type, since they're not using mocking, unless external resources in called. But since they do not depend on any repository, can be treated as unit tests. Due to great size of complete response from Apple Music Charts "songs" endpoint, only 10 tracks are stored in fixtures. Here there are their ids: ['1590036258', '1581180951', '1556097162', '1540314624', '1590029763', '1595158660', '1540157580', '1592740089', '1590029547', '1581532067'] Track with id '1581532067' in 'us_tracks.json' file has no artist attributes. """ from typing import Dict, List from unittest import mock from unittest.mock import Mock import awswrangler as wr import pytest from jsonschema.exceptions import ValidationError from slz_apple_music_charts_scrapper.entities import JobExecutionResult from slz_apple_music_charts_scrapper.exceptions import ( APIInvalidResponse, APINotFound, APIUnauthorized, APIUnavailable, ) from slz_apple_music_charts_scrapper.manager import ApplicationManager from tests.entities import BucketFilesCounter, BucketPathsTuple, CommonTestValues @pytest.mark.parametrize( 'track_ids, expected_keys', [ ([], []), (['1590036021', '1592774398', '1596505572'], []), # Unknown tracks (['1590036258', '1581180951', '1556097162'], ['1590036258', '1581180951', '1556097162']), (['1581532067'], ['1581532067']), ] ) def test__retrieve_additional_chart_data( application_manager_stateless: ApplicationManager, common_test_values: CommonTestValues, track_ids: List[str], expected_keys: List[str], ): result = application_manager_stateless._retrieve_additional_chart_data( storefront=common_test_values.test_storefront, tracks_ids=track_ids, ) assert set(expected_keys) == result.keys() def test__retrieve_additional_chart_data__artist_attributes( application_manager_stateless: ApplicationManager, common_test_values: CommonTestValues, ): # Check special track missing artis attributes. track_id = '1581532067' result = application_manager_stateless._retrieve_additional_chart_data( storefront=common_test_values.test_storefront, tracks_ids=[track_id], ) assert {track_id} == result.keys() assert result[track_id]['artists'][0]['genre_names'] == ['NA'] assert result[track_id]['artists'][0]['name'] == 'NA' assert result[track_id]['artists'][0]['url'] == 'NA' assert result[track_id]['artists'][0]['editorial_notes'] == { 'name': 'NA', 'short': 'NA', 'standard': 'NA', 'tagline': 'NA', } @pytest.mark.parametrize( 'chart_data, is_artists_fetched', [ ([], {}), ([{ 'id': '1590036258' }, { 'id': '1540157580' }], { '1590036258': True, '1540157580': True }), # Include the track that has no artist attributes. ([{ 'id': '1581532067' }, { 'id': '1590036258' }, { 'id': '1540157580' }], { '1581532067': True, '1590036258': True, '1540157580': True }), # Unknown tracks. ([{ 'id': '1590036021' }, { 'id': '1592774398' }, { 'id': '1596505572' }], { '1590036021': False, '1592774398': False, '1596505572': False }), ] ) def test__extend_chart_data( application_manager_stateless: ApplicationManager, common_test_values: CommonTestValues, chart_data: List[Dict[str, str]], is_artists_fetched: List[Dict[str, bool]], ): initial_track_number = len(chart_data) expected_artist_attributes = ['id', 'href', 'genre_names', 'name', 'url', 'editorial_notes'] result = application_manager_stateless._extend_chart_data( storefront=common_test_values.test_storefront, chart_data=chart_data ) assert chart_data is result # Warning! Object returned by this method is the same accepted obj. assert len(result) == initial_track_number assert all('artists' in track for track in result) assert all((len(track['artists']) != 0) is is_artists_fetched[track['id']] for track in result) assert all( keys in expected_artist_attributes for track in result for artist in track['artists'] for keys in artist ) def test___retrieve_chart_data( application_manager_stateless: ApplicationManager, common_test_values: CommonTestValues, ): additional_data_tracks = [ '1590036258', '1581180951', '1556097162', '1540314624', '1590029763', '1595158660', '1540157580', '1592740089', '1590029547', '1581532067' ] result = application_manager_stateless._retrieve_chart_data( storefront=common_test_values.test_storefront, source_storefront=common_test_values.test_storefront, source_playlist_id=common_test_values.test_playlist, ) assert all( track['id'] in additional_data_tracks and track['artists'] or track['id'] not in additional_data_tracks and not track['artists'] for track in result ) @pytest.mark.parametrize( 'is_hash_changed, is_valid, expected_counters, expected_result', [ ( True, True, BucketFilesCounter(0, 0, 1), JobExecutionResult(success=True, is_hash_updated=True, market=mock.ANY) ), ( True, False, BucketFilesCounter(0, 1, 0), JobExecutionResult(success=False, is_hash_updated=True, market=mock.ANY) ), # Here validator will be skipped ( False, True, BucketFilesCounter(0, 0, 0), JobExecutionResult(success=True, is_hash_updated=False, market=mock.ANY) ), # Here validator will be skipped ( False, False, BucketFilesCounter(0, 0, 0), JobExecutionResult(success=True, is_hash_updated=False, market=mock.ANY) ), ] ) def test_process_storefront_chart_data__hash_and_validate( application_manager_stateless: ApplicationManager, bucket_paths_stateless: BucketPathsTuple, common_test_values: CommonTestValues, is_hash_changed: bool, is_valid: bool, expected_counters: BucketFilesCounter, expected_result: JobExecutionResult, ): """Test various scenarios based on hash and data frame validator checks.""" app_manager = application_manager_stateless bucket_paths = bucket_paths_stateless # Check that initially buckets are empty. assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == 0 validator_mock_effect = {} if is_valid else {'side_effect': ValidationError('test')} app_manager._validator_service.validate_data_frame = Mock(**validator_mock_effect) app_manager._snapshot_service.is_hash_updated = Mock(return_value=is_hash_changed) result = app_manager.process_storefront_chart_data(common_test_values.test_storefront) assert expected_result == result assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == expected_counters.quarantine assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == expected_counters.corrupted assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == expected_counters.decompressed @pytest.mark.parametrize( 'exception', [ APIUnavailable, APIUnauthorized, APINotFound, APIInvalidResponse, ] ) def test_process_storefront_chart_data_playlist__source_fail( application_manager_stateless: ApplicationManager, common_test_values: CommonTestValues, bucket_paths_stateless: BucketPathsTuple, exception: Exception, ): """Test various scenarios happening during request send to the source of charts data.""" app_manager = application_manager_stateless bucket_paths = bucket_paths_stateless # Check that initially buckets are empty. assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == 0 app_manager._vendor_api_service.get_apple_playlist = Mock(side_effect=exception('test')) app_manager.process_storefront_chart_data(common_test_values.test_storefront) # Even raw data wasn't fetched yet. Nothing was saved, nothing moved. assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == 0 @pytest.mark.parametrize( 'exception', [ APIUnavailable, APIUnauthorized, APINotFound, APIInvalidResponse, ] ) def test_process_storefront_chart_data_songs__source_fail( application_manager_stateless: ApplicationManager, common_test_values: CommonTestValues, bucket_paths_stateless: BucketPathsTuple, exception: Exception, ): """Test various scenarios happening during request send to the source of charts data.""" app_manager = application_manager_stateless bucket_paths = bucket_paths_stateless # Check that initially buckets are empty. assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == 0 app_manager._vendor_api_service.get_apple_songs = Mock(side_effect=exception('test')) app_manager.process_storefront_chart_data(common_test_values.test_storefront) # Even raw data wasn't fetched yet. Nothing was saved, nothing moved. assert len(wr.s3.list_objects(path=bucket_paths.quarantine)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.corrupted)) == 0 assert len(wr.s3.list_objects(path=bucket_paths.decompressed)) == 0