# pylint: disable=redefined-outer-name,unused-argument,invalid-name,too-many-locals import json from contextlib import nullcontext as does_not_raise from datetime import timedelta from os.path import join as p_join from typing import Optional, Tuple, Union import pytest from _pytest.fixtures import FixtureRequest, SubRequest from _pytest.python_api import RaisesContext from dapd_db_schema.schemas import workflow as schema from smelog.factory import SmeBoundLogger from dapd_public_api_scraper.const import DSPEnum from dapd_public_api_scraper.entity.apple_music import Album, Artist, Playlist, Track from dapd_public_api_scraper.entity.base import Task from dapd_public_api_scraper.repository import postgres from dapd_public_api_scraper.repository.postgres import Postgres from dapd_public_api_scraper.service.workflowdb import apple_music from dapd_public_api_scraper.service.workflowdb.base import WorkflowDBConfigurationError from dapd_public_api_scraper.utils.misc import utcnow from tests.entities import AppleMusicMetaEntity, AppleMusicWFDBService, AppleMusicWFDBServiceClass from tests.utils import get_album, get_artist, get_playlist, get_storefront, get_track @pytest.fixture def init_storefronts(apple_music_workflowdb_playlist, db): data_source = apple_music_workflowdb_playlist.get_data_source() storefronts = [ get_storefront(id='us', data_source_id=data_source.id, name='US'), get_storefront(id='de', data_source_id=data_source.id, name='DE'), get_storefront(id='gb', data_source_id=data_source.id, name='GB'), get_storefront(id='it', data_source_id=data_source.id, name='IT'), get_storefront(id='mx', data_source_id=data_source.id, name='MX'), get_storefront(id='ae', data_source_id=data_source.id, name='AE'), ] for storefront in storefronts: db.session.add(storefront) db.session.flush() @pytest.fixture def data_source_test(db: postgres.Postgres) -> schema.DataSource: return db.session.query(schema.DataSource).filter(schema.DataSource.name == 'apple_music').one() @pytest.fixture def task_and_item__indirect( s3_backup_apple_music_dir: str, request: SubRequest, ) -> Tuple[Task, AppleMusicMetaEntity]: """Task entity accepting as a tuple two items: entity type and entity ID.""" entity_class, entity_type, entity_id = request.param with open(p_join(s3_backup_apple_music_dir, entity_type, entity_id), 'r') as _file: report = json.load(_file) item = entity_class.from_dict(report) return Task(id=item.meta.item_id, storefront=item.meta.item_storefront), item @pytest.mark.integration @pytest.mark.parametrize( 'service_class', [ apple_music.AppleMusicAlbum, apple_music.AppleMusicArtist, apple_music.AppleMusicPlaylist, apple_music.AppleMusicTrack, ] ) def test_get_data_source( logger: SmeBoundLogger, db: Postgres, service_class: AppleMusicWFDBServiceClass ): service: AppleMusicWFDBService = service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=None, ) item = service.get_data_source() assert item is not None assert item.name == DSPEnum.apple_music.value @pytest.mark.integration @pytest.mark.parametrize( 'test_ttl_value, expected, expectation', [ ('1_DAY', timedelta(days=1), does_not_raise()), ('30_DAYS', timedelta(days=30), does_not_raise()), ('365_DAYS', timedelta(days=365), does_not_raise()), # Next value is expected to be missing in the DB. ('ASHJDKGAS', timedelta(days=1), pytest.raises(WorkflowDBConfigurationError)), ], ) @pytest.mark.parametrize( 'service_class', [ apple_music.AppleMusicAlbum, apple_music.AppleMusicArtist, apple_music.AppleMusicPlaylist, apple_music.AppleMusicTrack, ], ) def test_init_and_get_is_removed_ttl( logger: SmeBoundLogger, db: Postgres, service_class: AppleMusicWFDBServiceClass, test_ttl_value: str, expected: timedelta, expectation: Union[does_not_raise, RaisesContext], ): service: Optional[AppleMusicWFDBService] = None with expectation: service = service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=test_ttl_value, ) # If service creation has failed, just leave. if service is None: return is_removed_ttl = service.get_is_removed_ttl() assert isinstance(is_removed_ttl, timedelta) assert is_removed_ttl == expected @pytest.mark.integration @pytest.mark.parametrize( 'test_ttl_value', ['1_DAY', '30_DAYS', '365_DAYS'], ) @pytest.mark.parametrize( 'service_class, task_and_item__indirect, model_fixture_name', [ ( apple_music.AppleMusicAlbum, (Album, 'album', '1031998253.json'), 'album_test', ), ( apple_music.AppleMusicArtist, (Artist, 'artist', '129045039.json'), 'artist_test', ), ( apple_music.AppleMusicPlaylist, (Playlist, 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb.json'), 'playlist_test', ), ( apple_music.AppleMusicTrack, (Track, 'track', '1126431409.json'), 'track_test', ), ], indirect=['task_and_item__indirect'] ) @pytest.mark.freeze_time('2021-12-01') def test_set_removed( logger: SmeBoundLogger, db: Postgres, init_storefronts: None, task_and_item__indirect: Tuple[Task, AppleMusicMetaEntity], model_fixture_name: str, service_class: AppleMusicWFDBServiceClass, test_ttl_value: str, request: FixtureRequest, ): InstanceType = Union[schema.Album, schema.Artist, schema.Playlist, schema.Track] instance: InstanceType = request.getfixturevalue(model_fixture_name) service = service_class( logger=logger, db=db, insert_batch_size=100, is_removed_ttl=test_ttl_value, ) task, _ = task_and_item__indirect assert instance.is_removed is False assert instance.is_removed_expired_at is None result = service.set_removed(task) assert result is True assert instance.is_removed is True assert instance.is_removed_expired_at == utcnow() + service.get_is_removed_ttl() @pytest.mark.integration def test_get_tracks_processed_recently( init_storefronts, apple_music_workflowdb_playlist, db, s3_backup_apple_music_dir, clean_db ): with open( p_join(s3_backup_apple_music_dir, 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb.json'), 'r' ) as fin: report = json.load(fin) item = Playlist.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = apple_music_workflowdb_playlist.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) playlist = get_playlist( name='pop', label='pop100', id=task.id, data_source_id=data_source.id, storefront_id=task.storefront, created_at=created_at, expired_at=expired_at, ) db.session.add(playlist) db.session.commit() now = utcnow() track = item.tracks[0] # new track, just created by playlist ingestion track = get_track( name=track['attributes']['name'], label=track['attributes']['name'], id=track['id'], data_source_id=data_source.id, created_at=now, updated_at=None, expired_at=now + timedelta(hours=4), ) playlist.tracks.append(track) db.session.commit() # track updated by playlist ingestion track = item.tracks[1] track = get_track( name=track['attributes']['name'], label=track['attributes']['name'], id=track['id'], data_source_id=data_source.id, created_at=now, updated_at=now + timedelta(hours=5), expired_at=now + timedelta(hours=4), ) playlist.tracks.append(track) db.session.commit() # track scheduled for update, but not updated yet track = item.tracks[2] track = get_track( name=track['attributes']['name'], label=track['attributes']['name'], id=track['id'], data_source_id=data_source.id, created_at=now, updated_at=now + timedelta(hours=4), expired_at=now + timedelta(hours=4), ) playlist.tracks.append(track) db.session.commit() tracks = apple_music_workflowdb_playlist.get_tracks_processed_recently(playlist) assert {track.id for track in tracks} == {item.tracks[0]['id'], item.tracks[1]['id']} @pytest.mark.integration def test_write_playlist( init_storefronts, apple_music_workflowdb_playlist, db, s3_backup_apple_music_dir, clean_db ): with open( p_join(s3_backup_apple_music_dir, 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb.json'), 'r' ) as fin: report = json.load(fin) item = Playlist.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = apple_music_workflowdb_playlist.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) playlist = get_playlist( name='pop', label='pop100', id=task.id, data_source_id=data_source.id, storefront_id=task.storefront, created_at=created_at, expired_at=expired_at, ) db.session.add(playlist) db.session.commit() api_track_ids = {x['id'] for x in item.tracks} assert apple_music_workflowdb_playlist.write(task, item) is True updated_playlist = apple_music_workflowdb_playlist.get(task.id, task.storefront) db_track_ids = {x.id for x in updated_playlist.tracks} assert api_track_ids == db_track_ids @pytest.mark.integration def test_write_track( init_storefronts, apple_music_workflowdb_track, db, s3_backup_apple_music_dir, clean_db ): with open(p_join(s3_backup_apple_music_dir, 'track', '1126431409.json'), 'r') as fin: report = json.load(fin) item = Track.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = apple_music_workflowdb_track.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) track = get_track( id=task.id, name=item.name, label=item.name, data_source_id=data_source.id, created_at=created_at, expired_at=expired_at, ) db.session.add(track) db.session.commit() api_album_ids = {x['id'] for x in item.albums} api_artist_ids = {x['id'] for x in item.artists} assert apple_music_workflowdb_track.write(task, item) is True updated_track = apple_music_workflowdb_track.get(task.id) db_album_ids = {x.id for x in updated_track.albums} assert api_album_ids == db_album_ids db_artist_ids = {x.id for x in updated_track.artists} assert api_artist_ids == db_artist_ids @pytest.mark.integration def test_write_album( init_storefronts, apple_music_workflowdb_album, db, s3_backup_apple_music_dir, clean_db ): with open(p_join(s3_backup_apple_music_dir, 'album', '1031998253.json'), 'r') as fin: report = json.load(fin) item = Album.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = apple_music_workflowdb_album.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) album = get_album( id=task.id, name=item.name, label=item.name, data_source_id=data_source.id, created_at=created_at, expired_at=expired_at, ) db.session.add(album) db.session.commit() api_artist_ids = {x['id'] for x in item.artists} assert apple_music_workflowdb_album.write(task, item) is True updated_album = apple_music_workflowdb_album.get(task.id) db_artist_ids = {x.id for x in updated_album.artists} assert api_artist_ids == db_artist_ids @pytest.mark.integration def test_write_artist( init_storefronts, apple_music_workflowdb_artist, db, s3_backup_apple_music_dir, clean_db ): with open(p_join(s3_backup_apple_music_dir, 'artist', '129045039.json'), 'r') as fin: report = json.load(fin) item = Artist.from_dict(report) task = Task(id=item.meta.item_id, storefront=item.meta.item_storefront) data_source = apple_music_workflowdb_artist.get_data_source() created_at = utcnow() expired_at = utcnow() + timedelta(hours=4) artist = get_artist( id=task.id, name=item.name, label=item.name, data_source_id=data_source.id, created_at=created_at, expired_at=expired_at, ) db.session.add(artist) db.session.commit() assert apple_music_workflowdb_artist.write(task, item) is True