# pylint: disable=too-many-locals,redefined-outer-name,protected-access,too-many-arguments import json from dataclasses import dataclass from typing import Type from unittest import mock import pytest from mypy_boto3_s3 import S3Client from dapd_public_api_scraper.config.entity import StorageConfig, Tasks from dapd_public_api_scraper.entity import apple_music as apple_music_entity from dapd_public_api_scraper.entity import spotify as spotify_entity from dapd_public_api_scraper.entity.base import Buffer, Item, Meta, Task from dapd_public_api_scraper.manager import SyncTaskManager from dapd_public_api_scraper.service.scraper import apple_music as apple_music_scraper from dapd_public_api_scraper.service.scraper import spotify as spotify_scraper from dapd_public_api_scraper.service.scraper.abs import Scraper from dapd_public_api_scraper.service.scraper.spotify import PlaylistScraper from dapd_public_api_scraper.service.storage import Storage from dapd_public_api_scraper.service.stream import Stream from dapd_public_api_scraper.service.workflowdb.spotify import SpotifyPlaylist @dataclass class CommonValues: raw_data_bucket: str = 'raw_data_bucket' corrupted_bucket: str = 'corrupted_bucket' @pytest.fixture def storage_config() -> StorageConfig: return StorageConfig( raw_data_bucket=CommonValues.raw_data_bucket, corrupted_bucket=CommonValues.corrupted_bucket, ) @pytest.fixture def init_buckets(s3_client: S3Client): s3_client.create_bucket(Bucket=CommonValues.raw_data_bucket) s3_client.create_bucket(Bucket=CommonValues.corrupted_bucket) @pytest.fixture def meta_fixture() -> Meta: return Meta( item_id='id', item_type='type', item_ts='', item_storefront='storefront', data_source='source', app_version='', ) @pytest.mark.parametrize( 'is_item_valid, is_item_fetched, expected_result, expected_file_path_state, ' 'expected_buff_call, expected_raw_files_number, expected_corrupted_files_number', [ (True, True, True, True, True, 1, 0), (False, True, False, False, False, 0, 1), (False, False, False, False, False, 0, 0), ] ) @pytest.mark.parametrize( 'item_class, scraper_class', [ (apple_music_entity.Album, apple_music_scraper.AlbumScraper), (apple_music_entity.Artist, apple_music_scraper.ArtistScraper), (apple_music_entity.Playlist, apple_music_scraper.PlaylistScraper), (apple_music_entity.Track, apple_music_scraper.TrackScraper), (spotify_entity.Album, spotify_scraper.AlbumScraper), (spotify_entity.Artist, spotify_scraper.ArtistScraper), (spotify_entity.Playlist, spotify_scraper.PlaylistScraper), (spotify_entity.Track, spotify_scraper.TrackScraper), ] ) def test__download( s3_client: S3Client, init_buckets: None, # pylint: disable=unused-argument storage_config: StorageConfig, meta_fixture: Meta, item_class: Type[Item], scraper_class: Type[Scraper], is_item_valid: bool, is_item_fetched: bool, expected_result: bool, expected_file_path_state: bool, expected_buff_call: bool, expected_raw_files_number: int, expected_corrupted_files_number: int, ): task = Task(id='abc', storefront='US') item = item_class( item={ 'id': 'item_id', 'data': [{ 'id': 'item_id' }], # Mimic filename source for known vendors. 'item': { 'id': 'item_id' }, }, meta=meta_fixture, ) if is_item_fetched else None scraper = mock.MagicMock(spec=scraper_class) scraper.get_item.return_value = task, item, is_item_valid buff = mock.MagicMock(spec=Buffer) storage = Storage( logger=mock.Mock(), config=storage_config, ) manager = SyncTaskManager( logger=mock.Mock(), task_source=Tasks(False, stdin=json.dumps([task.to_dict()])), concurrency=1, workflowdb=mock.Mock(), storage=storage, scraper=scraper, stream=mock.MagicMock(spec=Stream), buff=buff, s3_client=s3_client, ) is_success = manager._download(task) assert is_success == expected_result if is_item_fetched: # file path may be or may be not added during download assert bool(item.meta.file_path) is expected_file_path_state if expected_buff_call: buff.add.assert_called_once_with(task, item) else: buff.add.assert_not_called() raw_files = s3_client.list_objects(Bucket=CommonValues.raw_data_bucket) assert len(raw_files.get('Contents', [])) == expected_raw_files_number corrupted_files = s3_client.list_objects(Bucket=CommonValues.corrupted_bucket) assert len(corrupted_files.get('Contents', [])) == expected_corrupted_files_number def test_sync_manager(logger): raw_data_bucket = 'raw_data' upload_path = 'upload_path' task = Task(id='abc', storefront='US') tasks = Tasks(False, stdin=json.dumps([task.to_dict()])) workflowdb = mock.MagicMock(spec=SpotifyPlaylist) storage = mock.MagicMock(spec=Storage) scraper = mock.MagicMock(spec=PlaylistScraper) stream = mock.MagicMock(spec=Stream) buff = Buffer() s3_client = mock.Mock() # This item will be changed by the test run. That's why we need another expected item. api_item = Item( item={}, meta=Meta( item_id=task.id, item_type='playlist', item_ts='', item_storefront=task.storefront, data_source='spotify', app_version='', ) ) expected_api_item = Item( item={}, meta=Meta( item_id=task.id, item_type='playlist', item_ts='', item_storefront=task.storefront, data_source='spotify', app_version='', ) ) scraper.get_item.return_value = (None, api_item, True) storage.get_raw_data_storage_root.return_value = raw_data_bucket storage.get_file_path_for_item.return_value = upload_path storage.upload_bytes.return_value = True stream.write.return_value = [ [(task, api_item)], ] manager = SyncTaskManager( logger, tasks, concurrency=1, workflowdb=workflowdb, storage=storage, scraper=scraper, stream=stream, buff=buff, s3_client=s3_client, ) manager.load_tasks() assert manager.tasks, [task] manager.process() scraper.get_item.assert_called_with(task) storage.get_file_path_for_item.assert_called_with(raw_data_bucket, api_item) storage.upload_bytes.assert_called_with(upload_path, expected_api_item.to_backup_message()) stream.write.assert_called_with(buff) assert api_item.meta.file_path == upload_path expected_buff = Buffer() expected_buff.add(task, api_item) with buff.data() as data: with expected_buff.data() as expected_data: assert data == expected_data workflowdb.write.assert_called_with(task, api_item) manager.cleanup() @pytest.mark.skip(reason='Need to be implemented') def test_async_manager(): assert True