# pylint: disable=redefined-outer-name,unused-argument,protected-access,too-many-locals # pylint: disable=too-many-arguments,fixme import json from datetime import timedelta from http import HTTPStatus from os.path import join as p_join from typing import Optional, Tuple, Type from unittest import mock import pytest from _pytest.fixtures import FixtureRequest, SubRequest from dapd_db_schema.schemas import workflow as schema from requests import HTTPError, Response from smelog.factory import SmeBoundLogger from dapd_public_api_scraper import const from dapd_public_api_scraper.entity import apple_music as entity from dapd_public_api_scraper.entity.base import Meta, Task from dapd_public_api_scraper.repository import postgres from dapd_public_api_scraper.repository.postgres import Postgres from dapd_public_api_scraper.repository.validator import Validator from dapd_public_api_scraper.service.scraper import apple_music as scraper from dapd_public_api_scraper.service.workflowdb import apple_music as db_service from dapd_public_api_scraper.utils.misc import utcnow from tests.entities import ( AppleMusicMetaEntity, AppleMusicScraper, AppleMusicScraperClass, AppleMusicWFDBService, AppleMusicWFDBServiceClass, ) from tests.utils import get_storefront pytestmark = pytest.mark.integration TaskAndItemResponse = Tuple[Task, AppleMusicMetaEntity, Type[AppleMusicMetaEntity], str] @pytest.fixture def data_source_test(db: postgres.Postgres) -> schema.DataSource: return db.session.query(schema.DataSource).filter(schema.DataSource.name == 'apple_music').one() @pytest.fixture def init_storefronts(db, data_source_test): storefronts = [ get_storefront(id='us', data_source_id=data_source_test.id, name='US'), get_storefront(id='de', data_source_id=data_source_test.id, name='DE'), get_storefront(id='gb', data_source_id=data_source_test.id, name='GB'), get_storefront(id='it', data_source_id=data_source_test.id, name='IT'), get_storefront(id='mx', data_source_id=data_source_test.id, name='MX'), get_storefront(id='ae', data_source_id=data_source_test.id, name='AE'), ] for storefront in storefronts: db.session.add(storefront) db.session.commit() yield db.session.query(schema.Storefront).delete() db.session.commit() # TODO: fix multiple fixtures used in the same fashion @pytest.fixture def task_and_item__indirect( s3_backup_apple_music_dir: str, request: SubRequest, ) -> TaskAndItemResponse: """Task entity accepting as a tuple two items: entity type and entity ID.""" entity_class: Type[AppleMusicMetaEntity] entity_type: str entity_id: str entity_class, entity_type, entity_id = request.param with open( p_join(s3_backup_apple_music_dir, entity_type, entity_id), 'r', encoding='utf-8' ) as _file: report = json.load(_file) item = entity_class.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) return task, item, entity_class, entity_type @pytest.fixture def validator_service__indirect_schema( schemas_apple_music_dir: str, request: SubRequest ) -> Validator: validator = Validator(logger=mock.Mock()) schema_path = f'{schemas_apple_music_dir}/{request.param}' validator.load_from_fs(schema_path) return validator @pytest.mark.parametrize( 'scraper_class, db_service_class, task_and_item__indirect, ' 'provider_response, fixture_name', [ ( scraper.AlbumScraper, db_service.AppleMusicAlbum, (entity.Album, 'album', '1031998253.json'), { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, None, ), ( scraper.ArtistScraper, db_service.AppleMusicArtist, (entity.Artist, 'artist', '129045039.json'), { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, None, ), ( scraper.PlaylistScraper, db_service.AppleMusicPlaylist, (entity.Playlist, 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb.json'), # snapshot ID is '7146b80182e268cf9f7d3e81794c830547cac1eab4d7feee70cfb2fd1ad7f928' { 'data': [{ 'id': 'abc', 'attributes': { 'url': 'test_url', 'lastModifiedDate': 'abc' } }] }, 'playlist_test', ), ( scraper.TrackScraper, db_service.AppleMusicTrack, (entity.Track, 'track', '1126431409.json'), { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, None, ), ], indirect=['task_and_item__indirect'], ) def test__get_item__success( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, task_and_item__indirect: TaskAndItemResponse, scraper_class: AppleMusicScraperClass, db_service_class: AppleMusicWFDBServiceClass, provider_response: dict, fixture_name: str, request: FixtureRequest, ): if fixture_name: request.getfixturevalue(fixture_name) task, *_ = task_and_item__indirect db_service_instance: AppleMusicWFDBService = db_service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=None, ) scraper_class.get_client = mock.Mock() scraper_class._call = mock.Mock(return_value=provider_response) scraper_instance: AppleMusicScraper = scraper_class( logger=logger, workflowdb=db_service_instance, validator=mock.Mock(), secrets_manager=mock.Mock(), credentials=mock.Mock(), ) result_task, result_item, is_item_valid = scraper_instance.get_item(task=task) assert result_item is not None assert result_task is task assert is_item_valid is True @pytest.mark.parametrize( 'scraper_class, db_service_class, task_and_item__indirect,' 'fixture_name, provider_response', [ ( scraper.AlbumScraper, db_service.AppleMusicAlbum, (entity.Album, 'album', '1031998253.json'), 'album_test', { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, ), ( scraper.ArtistScraper, db_service.AppleMusicArtist, (entity.Artist, 'artist', '129045039.json'), 'artist_test', { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, ), ( scraper.PlaylistScraper, db_service.AppleMusicPlaylist, (entity.Playlist, 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb.json'), 'playlist_test', # snapshot ID is '7146b80182e268cf9f7d3e81794c830547cac1eab4d7feee70cfb2fd1ad7f928' { 'data': [{ 'id': 'abc', 'attributes': { 'url': 'test_url', 'lastModifiedDate': 'abc' } }] }, ), ( scraper.TrackScraper, db_service.AppleMusicTrack, (entity.Track, 'track', '1126431409.json'), 'track_test', { 'data': [{ 'attributes': { 'url': 'test_url' } }] }, ), ], indirect=['task_and_item__indirect'], ) @pytest.mark.freeze_time('2021-12-01') def test__get_item__fail_not_found( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, task_and_item__indirect: TaskAndItemResponse, scraper_class: AppleMusicScraperClass, db_service_class: AppleMusicWFDBServiceClass, fixture_name: Optional[str], provider_response: dict, request: FixtureRequest, ): instance = request.getfixturevalue(fixture_name) task, *_ = task_and_item__indirect db_service_instance: AppleMusicWFDBService = db_service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl='1_DAY', ) response = Response() response.status_code = HTTPStatus.NOT_FOUND exception = HTTPError('test') exception.response = response scraper_class._call = mock.Mock(side_effect=exception) scraper_class.get_client = mock.Mock() scraper: AppleMusicScraper = scraper_class( logger=logger, workflowdb=db_service_instance, validator=mock.Mock(), secrets_manager=mock.Mock(), credentials=mock.Mock(), ) assert instance.is_removed is False assert instance.is_removed_expired_at is None result_task, result_item, is_item_valid = scraper.get_item(task=task) assert result_item is None assert result_task is task assert is_item_valid is False assert instance.is_removed is True assert instance.is_removed_expired_at == utcnow() + timedelta(days=1) @pytest.mark.parametrize( 'scraper_class, db_service_class, task_and_item__indirect,' 'fixture_name, validator_service__indirect_schema', [ ( scraper.AlbumScraper, db_service.AppleMusicAlbum, (entity.Album, 'album', '1031998253.json'), 'album_test', 'album/2021-12-13.json', ), ( scraper.ArtistScraper, db_service.AppleMusicArtist, (entity.Artist, 'artist', '129045039.json'), 'artist_test', 'artist/2021-12-13.json', ), ( scraper.PlaylistScraper, db_service.AppleMusicPlaylist, (entity.Playlist, 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb.json'), 'playlist_test', 'playlist/2021-12-13.json', ), ( scraper.TrackScraper, db_service.AppleMusicTrack, (entity.Track, 'track', '1126431409.json'), 'track_test', 'track/2021-12-13.json', ), ], indirect=['task_and_item__indirect', 'validator_service__indirect_schema'], ) @pytest.mark.freeze_time('2021-12-01') def test__get_item__fail_validation( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, task_and_item__indirect: TaskAndItemResponse, scraper_class: AppleMusicScraperClass, db_service_class: AppleMusicWFDBServiceClass, fixture_name: Optional[str], validator_service__indirect_schema: Validator, request: FixtureRequest, ): request.getfixturevalue(fixture_name) task, _, item_class, item_type = task_and_item__indirect invalid_item_content = {'response': 'dummy'} expected_item = item_class( item=invalid_item_content, meta=Meta( item_id=task.id, item_type=item_type, item_ts=utcnow().isoformat(), item_storefront=task.storefront, data_source='apple_music', app_version=const.VERSION, ) ) db_service_instance: AppleMusicWFDBService = db_service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl='1_DAY', ) scraper_class._call = mock.Mock(return_value=invalid_item_content) scraper_class.get_client = mock.Mock() scraper_instance: AppleMusicScraper = scraper_class( logger=logger, workflowdb=db_service_instance, validator=validator_service__indirect_schema, secrets_manager=mock.Mock(), credentials=mock.Mock(), ) result_task, result_item, is_item_valid = scraper_instance.get_item(task=task) assert result_item is not None assert result_item.to_dict() == expected_item.to_dict() assert result_task is task assert is_item_valid is False