# pylint: disable=redefined-outer-name,unused-argument,invalid-name,too-many-locals # pylint: disable=too-many-arguments import json from contextlib import nullcontext as does_not_raise from datetime import timedelta from os.path import join as p_join from typing import Optional, Tuple, Union import pytest from _pytest.fixtures import FixtureRequest, SubRequest from _pytest.python_api import RaisesContext from dapd_db_schema.schemas import workflow as schema from smelog.factory import SmeBoundLogger from dapd_public_api_scraper.const import DSPEnum from dapd_public_api_scraper.entity.base import Task from dapd_public_api_scraper.entity.spotify import Album, Artist, Playlist, Track from dapd_public_api_scraper.repository import postgres from dapd_public_api_scraper.repository.postgres import Postgres from dapd_public_api_scraper.service.workflowdb import spotify from dapd_public_api_scraper.service.workflowdb.base import WorkflowDBConfigurationError from dapd_public_api_scraper.utils.misc import utcnow from tests.entities import SpotifyMetaEntity, SpotifyWFDBService, SpotifyWFDBServiceClass from tests.utils import get_album, get_artist, get_playlist, get_storefront, get_track @pytest.fixture def init_storefronts(spotify_workflowdb_playlist, db): data_source = spotify_workflowdb_playlist.get_data_source() storefronts = [ get_storefront(id='global', data_source_id=data_source.id, name='Global'), ] for storefront in storefronts: db.session.add(storefront) db.session.flush() @pytest.fixture def data_source_test(db: postgres.Postgres) -> schema.DataSource: return db.session.query(schema.DataSource).filter(schema.DataSource.name == 'spotify').one() @pytest.fixture def task_and_item__indirect( s3_backup_spotify_dir: str, request: SubRequest, ) -> Tuple[Task, SpotifyMetaEntity]: """Task entity accepting as a tuple two items: entity type and entity ID.""" entity_class, entity_type, entity_id = request.param with open(p_join(s3_backup_spotify_dir, entity_type, entity_id), 'r') as _file: report = json.load(_file) item = entity_class.from_dict(report) return Task(id=item.meta.item_id, storefront=item.meta.item_storefront), item @pytest.mark.integration def test_get_data_source(spotify_workflowdb_playlist): item = spotify_workflowdb_playlist.get_data_source() assert item is not None assert item.name == DSPEnum.spotify.value @pytest.mark.integration @pytest.mark.parametrize( 'test_ttl_value, expected, expectation', [ ('1_DAY', timedelta(days=1), does_not_raise()), ('30_DAYS', timedelta(days=30), does_not_raise()), ('365_DAYS', timedelta(days=365), does_not_raise()), # Next value is expected to be missing in the DB. ('ASHJDKGAS', timedelta(days=1), pytest.raises(WorkflowDBConfigurationError)), ], ) @pytest.mark.parametrize( 'service_class', [ spotify.SpotifyAlbum, spotify.SpotifyArtist, spotify.SpotifyPlaylist, spotify.SpotifyTrack, ], ) def test_init_and_get_is_removed_ttl( logger: SmeBoundLogger, db: Postgres, service_class: SpotifyWFDBServiceClass, test_ttl_value: str, expected: timedelta, expectation: Union[does_not_raise, RaisesContext], ): service: Optional[SpotifyWFDBService] = None with expectation: service = service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=test_ttl_value, ) # If service creation has failed, just leave. if service is None: return is_removed_ttl = service.get_is_removed_ttl() assert isinstance(is_removed_ttl, timedelta) assert is_removed_ttl == expected @pytest.mark.integration @pytest.mark.parametrize( 'test_ttl_value', ['1_DAY', '30_DAYS', '365_DAYS'], ) @pytest.mark.parametrize( 'service_class, task_and_item__indirect, model_fixture_name', [ ( spotify.SpotifyAlbum, (Album, 'album', '0GHyMOR7XVVIU0NwQSrw90.json'), 'album_test', ), ( spotify.SpotifyArtist, (Artist, 'artist', '0cQuYRSzlItquYxsQKDvVc.json'), 'artist_test', ), ( spotify.SpotifyPlaylist, (Playlist, 'playlist', '034aPYRs3TW4tEpsDyokB6.json'), 'playlist_test', ), ( spotify.SpotifyTrack, (Track, 'track', '0E8qIw6KblOtDQYzzHQrxL.json'), 'track_test', ), ], indirect=['task_and_item__indirect'] ) @pytest.mark.freeze_time('2021-12-01') def test_set_removed( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, task_and_item__indirect: Tuple[Task, SpotifyMetaEntity], model_fixture_name: str, service_class: SpotifyWFDBServiceClass, test_ttl_value: str, request: FixtureRequest, ): InstanceType = Union[schema.Album, schema.Artist, schema.Playlist, schema.Track] instance: InstanceType = request.getfixturevalue(model_fixture_name) service = service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=test_ttl_value, ) task, _ = task_and_item__indirect assert instance.is_removed is False assert instance.is_removed_expired_at is None result = service.set_removed(task) assert result is True assert instance.is_removed is True assert instance.is_removed_expired_at == utcnow() + service.get_is_removed_ttl() @pytest.mark.integration def test_write_playlist( init_storefronts, spotify_workflowdb_playlist, db, s3_backup_spotify_dir, clean_db ): with open(p_join(s3_backup_spotify_dir, 'playlist', '034aPYRs3TW4tEpsDyokB6.json'), 'r') as fin: report = json.load(fin) item = Playlist.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = spotify_workflowdb_playlist.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) playlist = get_playlist( name='pop', label='pop100', id=task.id, data_source_id=data_source.id, storefront_id=task.storefront, created_at=created_at, expired_at=expired_at, ) db.session.add(playlist) db.session.commit() db.session.expire_all() api_track_ids = {x['track']['id'] for x in item.tracks} assert spotify_workflowdb_playlist.write(task, item) is True updated_playlist = spotify_workflowdb_playlist.get(task.id, task.storefront) db_track_ids = {x.id for x in updated_playlist.tracks} assert api_track_ids == db_track_ids @pytest.mark.integration def test_write_track( init_storefronts, spotify_workflowdb_track, db, s3_backup_spotify_dir, clean_db ): with open(p_join(s3_backup_spotify_dir, 'track', '0E8qIw6KblOtDQYzzHQrxL.json'), 'r') as fin: report = json.load(fin) item = Track.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = spotify_workflowdb_track.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) track = get_track( id=task.id, name=item.name, label=item.name, data_source_id=data_source.id, created_at=created_at, expired_at=expired_at, ) db.session.add(track) db.session.commit() api_album_id = item.album['id'] api_artist_ids = {x['id'] for x in item.artists} assert spotify_workflowdb_track.write(task, item) is True updated_track = spotify_workflowdb_track.get(task.id) db_album_ids = {x.id for x in updated_track.albums} assert len(db_album_ids) == 1 assert api_album_id == db_album_ids.pop() db_artist_ids = {x.id for x in updated_track.artists} assert api_artist_ids == db_artist_ids @pytest.mark.integration def test_write_album( init_storefronts, spotify_workflowdb_album, db, s3_backup_spotify_dir, clean_db ): with open(p_join(s3_backup_spotify_dir, 'album', '0GHyMOR7XVVIU0NwQSrw90.json'), 'r') as fin: report = json.load(fin) item = Album.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = spotify_workflowdb_album.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) album = get_album( id=task.id, name=item.name, label=item.name, data_source_id=data_source.id, created_at=created_at, expired_at=expired_at, ) db.session.add(album) db.session.commit() api_artist_ids = {x['id'] for x in item.artists} assert spotify_workflowdb_album.write(task, item) is True updated_album = spotify_workflowdb_album.get(task.id) db_artist_ids = {x.id for x in updated_album.artists} assert api_artist_ids == db_artist_ids @pytest.mark.integration def test_write_artist( init_storefronts, spotify_workflowdb_artist, db, s3_backup_spotify_dir, clean_db ): with open(p_join(s3_backup_spotify_dir, 'artist', '0cQuYRSzlItquYxsQKDvVc.json'), 'r') as fin: report = json.load(fin) item = Artist.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = spotify_workflowdb_artist.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) artist = get_artist( id=task.id, name=item.name, label=item.name, data_source_id=data_source.id, created_at=created_at, expired_at=expired_at, ) db.session.add(artist) db.session.commit() assert spotify_workflowdb_artist.write(task, item) is True