# pylint: disable=redefined-outer-name,unused-argument,protected-access,too-many-locals # pylint: disable=too-many-arguments,fixme import json from dataclasses import dataclass from datetime import timedelta from http import HTTPStatus from os.path import join as p_join from typing import Optional, Tuple, Type, Union from unittest import mock import pytest from _pytest.fixtures import FixtureRequest, SubRequest from dapd_db_schema.schemas import workflow as schema from smelog.factory import SmeBoundLogger from spotipy import SpotifyException from dapd_public_api_scraper import const from dapd_public_api_scraper.entity import spotify as entity from dapd_public_api_scraper.entity.base import Item, Meta, PlaylistMeta, Task from dapd_public_api_scraper.repository import postgres from dapd_public_api_scraper.repository.postgres import Postgres from dapd_public_api_scraper.repository.validator import Validator from dapd_public_api_scraper.service.scraper import spotify as scraper from dapd_public_api_scraper.service.workflowdb import spotify as db_service from dapd_public_api_scraper.utils.misc import utcnow from tests.entities import ( SpotifyMetaEntity, SpotifyScraper, SpotifyScraperClass, SpotifyWFDBService, SpotifyWFDBServiceClass, ) from tests.utils import get_json_from_path, get_storefront pytestmark = pytest.mark.integration TaskAndItemResponse = Tuple[Task, SpotifyMetaEntity, Type[SpotifyMetaEntity], str] @dataclass class ItemData: klass: Type[Item] type: str # TODO: fix multiple fixtures used in the same fashion @pytest.fixture def data_source_test(db: postgres.Postgres) -> schema.DataSource: return db.session.query(schema.DataSource).filter(schema.DataSource.name == 'spotify').one() @pytest.fixture def init_storefronts(data_source_test, db): storefronts = [ get_storefront(id='global', data_source_id=data_source_test.id, name='Global'), ] for storefront in storefronts: db.session.add(storefront) db.session.commit() yield db.session.query(schema.Storefront).delete() db.session.commit() @pytest.fixture def task_and_item__indirect( s3_backup_spotify_dir: str, request: SubRequest, ) -> TaskAndItemResponse: """Task entity accepting as a tuple two items: entity type and entity ID.""" entity_class: Type[SpotifyMetaEntity] entity_type: str entity_id: str entity_class, entity_type, entity_id = request.param with open(p_join(s3_backup_spotify_dir, entity_type, entity_id), 'r', encoding='utf-8') as file: report = json.load(file) item = entity_class.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) return task, item, entity_class, entity_type @pytest.fixture def validator_service__indirect_schema(schemas_spotify_dir: str, request: SubRequest) -> Validator: validator = Validator(logger=mock.Mock()) schema_path = f'{schemas_spotify_dir}/{request.param}' validator.load_from_fs(schema_path) return validator @pytest.mark.parametrize( 'scraper_class, db_service_class, task_and_item__indirect, ' 'fixture_name, validator_service__indirect_schema, raw_data_relative_path', [ ( scraper.AlbumScraper, db_service.SpotifyAlbum, (entity.Album, 'album', '0GHyMOR7XVVIU0NwQSrw90.json'), None, 'album/2021-12-13.json', 'album/0GHyMOR7XVVIU0NwQSrw90_valid.json', ), ( scraper.ArtistScraper, db_service.SpotifyArtist, (entity.Artist, 'artist', '0cQuYRSzlItquYxsQKDvVc.json'), None, 'artist/2021-12-13.json', 'artist/0cQuYRSzlItquYxsQKDvVc_valid.json', ), ( scraper.PlaylistScraper, db_service.SpotifyPlaylist, (entity.Playlist, 'playlist', '034aPYRs3TW4tEpsDyokB6.json'), 'playlist_test', 'playlist/2021-12-13.json', # TODO: Personalization should be tested separately. 'playlist/1T0WUWrDqIGt7OvHNA0amF_valid.json', ), ( scraper.TrackScraper, db_service.SpotifyTrack, (entity.Track, 'track', '0E8qIw6KblOtDQYzzHQrxL.json'), None, 'track/2021-12-13.json', 'track/0E8qIw6KblOtDQYzzHQrxL_valid.json', ), ], indirect=['task_and_item__indirect', 'validator_service__indirect_schema'], ) def test__get_item__success( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, raw_data_spotify_dir: str, task_and_item__indirect: TaskAndItemResponse, scraper_class: SpotifyScraperClass, db_service_class: SpotifyWFDBServiceClass, fixture_name: str, validator_service__indirect_schema: Validator, raw_data_relative_path: str, request: FixtureRequest, ): if fixture_name: request.getfixturevalue(fixture_name) task, *_ = task_and_item__indirect db_service_instance: SpotifyWFDBService = db_service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=None, ) scraper_class.api_client = mock.Mock() scraper_class.user_api_client = mock.Mock() scraper_class.get_client = mock.Mock() provider_response = get_json_from_path(p_join(raw_data_spotify_dir, raw_data_relative_path)) scraper_class._call = mock.Mock(return_value=provider_response) scraper_instance: SpotifyScraper = scraper_class( logger=logger, workflowdb=db_service_instance, validator=validator_service__indirect_schema, secrets_manager=mock.Mock(), credentials=mock.Mock(), ) result_task, result_item, is_item_valid = scraper_instance.get_item(task=task) assert result_item is not None assert result_task is task assert is_item_valid is True @pytest.mark.parametrize( 'scraper_class, db_service_class, task_and_item__indirect,' 'fixture_name, provider_response', [ ( scraper.AlbumScraper, db_service.SpotifyAlbum, (entity.Album, 'album', '0GHyMOR7XVVIU0NwQSrw90.json'), 'album_test', { 'external_urls': { 'spotify': 'test_data' } }, ), ( scraper.ArtistScraper, db_service.SpotifyArtist, (entity.Artist, 'artist', '0cQuYRSzlItquYxsQKDvVc.json'), 'artist_test', { 'external_urls': { 'spotify': 'test_data' } }, ), ( scraper.PlaylistScraper, db_service.SpotifyPlaylist, (entity.Playlist, 'playlist', '034aPYRs3TW4tEpsDyokB6.json'), 'playlist_test', { 'external_urls': { 'spotify': 'test_data' }, 'snapshot_id': '123', 'owner': { 'id': 'abc' }, 'name': '123', }, ), ( scraper.TrackScraper, db_service.SpotifyTrack, (entity.Track, 'track', '0E8qIw6KblOtDQYzzHQrxL.json'), 'track_test', { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, ), ], indirect=['task_and_item__indirect'], ) @pytest.mark.freeze_time('2021-12-01') def test__get_item__fail_not_found( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, task_and_item__indirect: TaskAndItemResponse, scraper_class: SpotifyScraperClass, db_service_class: SpotifyWFDBServiceClass, fixture_name: Optional[str], provider_response: dict, request: FixtureRequest, ): instance = request.getfixturevalue(fixture_name) task, *_ = task_and_item__indirect db_service_instance: SpotifyWFDBService = db_service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl='1_DAY', ) exception = SpotifyException(http_status=HTTPStatus.NOT_FOUND, code='test', msg='test') scraper_class._call = mock.Mock(side_effect=exception) scraper_class.get_client = mock.Mock() scraper_instance: SpotifyWFDBService = scraper_class( logger=logger, workflowdb=db_service_instance, validator=mock.Mock(), secrets_manager=mock.Mock(), credentials=mock.Mock(), ) assert instance.is_removed is False assert instance.is_removed_expired_at is None result_task, result_item, is_item_valid = scraper_instance.get_item(task=task) assert result_item is None assert result_task is task assert is_item_valid is False assert instance.is_removed is True assert instance.is_removed_expired_at == utcnow() + timedelta(days=1) @pytest.mark.parametrize( 'scraper_class, db_service_class, meta_class, task_and_item__indirect,' 'fixture_name, validator_service__indirect_schema', [ ( scraper.AlbumScraper, db_service.SpotifyAlbum, Meta, (entity.Album, 'album', '0GHyMOR7XVVIU0NwQSrw90.json'), 'album_test', 'album/2021-12-13.json', ), ( scraper.ArtistScraper, db_service.SpotifyArtist, Meta, (entity.Artist, 'artist', '0cQuYRSzlItquYxsQKDvVc.json'), 'artist_test', 'artist/2021-12-13.json', ), ( scraper.PlaylistScraper, db_service.SpotifyPlaylist, PlaylistMeta, (entity.Playlist, 'playlist', '034aPYRs3TW4tEpsDyokB6.json'), 'playlist_test', 'playlist/2021-12-13.json', ), ( scraper.TrackScraper, db_service.SpotifyTrack, Meta, (entity.Track, 'track', '0E8qIw6KblOtDQYzzHQrxL.json'), 'track_test', 'track/2021-12-13.json', ), ], indirect=['task_and_item__indirect', 'validator_service__indirect_schema'], ) @pytest.mark.freeze_time('2021-12-01') def test__get_item__fail_validation( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, meta_class: Type[Union[Meta, PlaylistMeta]], task_and_item__indirect: TaskAndItemResponse, scraper_class: SpotifyScraperClass, db_service_class: SpotifyWFDBServiceClass, fixture_name: Optional[str], validator_service__indirect_schema: Validator, request: FixtureRequest, ): request.getfixturevalue(fixture_name) task, _, item_class, item_type = task_and_item__indirect invalid_item_content = {'href': 123} expected_item = item_class( item=invalid_item_content, meta=meta_class( item_id=task.id, item_type=item_type, item_ts=utcnow().isoformat(), item_storefront=task.storefront, data_source='spotify', app_version=const.VERSION, ), ) db_service_instance: SpotifyWFDBService = db_service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl='1_DAY', ) scraper_class._call = mock.Mock(return_value=invalid_item_content) scraper_class.api_client = mock.Mock() scraper_class.user_api_client = mock.Mock() scraper_instance: SpotifyScraper = scraper_class( logger=logger, workflowdb=db_service_instance, validator=validator_service__indirect_schema, secrets_manager=mock.Mock(), credentials=mock.Mock(), ) result_task, result_item, is_item_valid = scraper_instance.get_item(task=task) assert result_item is not None assert result_item.to_dict() == expected_item.to_dict() assert result_task is task assert is_item_valid is False