# pylint: disable=redefined-outer-name,invalid-name import json import typing as t from concurrent.futures import ALL_COMPLETED, ThreadPoolExecutor, wait from os.path import join as p_join import pytest from _pytest.fixtures import FixtureRequest from smelog.factory import SmeBoundLogger from dapd_public_api_scraper.entity import apple_music, base, spotify ItemClass = t.TypeVar('ItemClass', bound=base.Item) @pytest.mark.parametrize( 'item_id,data,expected', [ ('abc111', 'body', 10), ('pl.123456', '{"data": "bytes"}', 26), ('', '', 0), ('a', '', 1), ('', 'a', 1), ] ) def test_stream_item(item_id, data, expected): assert base.StreamItem(id=item_id, data=data).size == expected def test_buffer(): buff = base.Buffer() with buff.data() as result: assert result == [] test_input = ( base.Task('pl.1', 'us'), base.Item( meta=base.Meta( 'pl.1', 'playlist', 'timestamp', 'us', 'spotify', 'v1', 'https://spotify.com/pl.1' ) ) ) buff.add(*test_input) with buff.data() as result: assert test_input == result[0] buff.clean() with buff.data() as result: assert result == [] def test_buffer_async(): buff = base.Buffer() test_data = [( base.Task('pl.{}'.format(x), 'us'), base.Item( meta=base.Meta( 'pl.{}'.format(x), 'playlist', 'ts', 'us', 'yandex.music', 'v1', 'https://music.ya.ru/pl.{}'.format(x) ) ) ) for x in range(9999)] def fn(item, buffer): buffer.add(item[0], item[1]) with ThreadPoolExecutor(max_workers=50) as pool: fs = [] for item in test_data: f = pool.submit(fn, item, buff) fs.append(f) wait(fs, return_when=ALL_COMPLETED) with buff.data() as result: assert len(result) == len(test_data) assert set(result) == set(test_data) @pytest.mark.parametrize( 'item_cls,path,item_id,name', [( apple_music.Playlist, 'playlist/pl.3343f35d630744efa7e01543e415ffeb.json', 'pl.3343f35d630744efa7e01543e415ffeb', 'Julie Adenuga: November 3, 2015', ), ( apple_music.Album, 'album/1031998253.json', '1031998253', 'Till the Wheels Come Off', ), ( apple_music.Artist, 'artist/129045039.json', '129045039', 'Zac Brown Band', ), ( apple_music.Track, 'track/1126431409.json', '1126431409', 'Ghetto Walkin\'', )] ) def test_apple_music_item(s3_backup_apple_music_dir, item_cls, path, item_id, name): with open(p_join(s3_backup_apple_music_dir, path), 'r') as f: backup_report = json.load(f) report = backup_report['item'] item = item_cls(meta=base.Meta.empty(), item=report) assert item.id == item_id assert item.name == name @pytest.mark.parametrize( 'item_cls,path,item_id,name', [ ( spotify.Playlist, 'playlist/034aPYRs3TW4tEpsDyokB6.json', '034aPYRs3TW4tEpsDyokB6', 'Jesuis', ), ( spotify.Album, 'album/0GHyMOR7XVVIU0NwQSrw90.json', '0GHyMOR7XVVIU0NwQSrw90', 'Fly', ), ( spotify.Artist, 'artist/0cQuYRSzlItquYxsQKDvVc.json', '0cQuYRSzlItquYxsQKDvVc', 'The Guess Who', ), ( spotify.Track, 'track/0E8qIw6KblOtDQYzzHQrxL.json', '0E8qIw6KblOtDQYzzHQrxL', 'Como TĂș (Magic Music Box)', ), ] ) def test_spotify_item(s3_backup_spotify_dir, item_cls, path, item_id, name): with open(p_join(s3_backup_spotify_dir, path), 'r') as f: backup_report = json.load(f) report = backup_report['item'] item = item_cls(meta=base.Meta.empty(), item=report) assert item.id == item_id assert item.name == name @pytest.mark.parametrize( 'item_class, root_path_fixture, path', [ (spotify.Album, 's3_backup_spotify_dir', 'album/0GHyMOR7XVVIU0NwQSrw90.json'), (spotify.Artist, 's3_backup_spotify_dir', 'artist/0cQuYRSzlItquYxsQKDvVc.json'), (spotify.Playlist, 's3_backup_spotify_dir', 'playlist/034aPYRs3TW4tEpsDyokB6.json'), (spotify.Track, 's3_backup_spotify_dir', 'track/0E8qIw6KblOtDQYzzHQrxL.json'), (apple_music.Album, 's3_backup_apple_music_dir', 'album/1031998253.json'), (apple_music.Artist, 's3_backup_apple_music_dir', 'artist/129045039.json'), (apple_music.Track, 's3_backup_apple_music_dir', 'track/1126431409.json'), ( apple_music.Playlist, 's3_backup_apple_music_dir', 'playlist/pl.3343f35d630744efa7e01543e415ffeb.json' ), ] ) # yapf: disable def test__item__to_transform_message( logger: SmeBoundLogger, request: FixtureRequest, item_class: t.Type[ItemClass], root_path_fixture: str, path: str, ): root_path = request.getfixturevalue(root_path_fixture) with open(p_join(root_path, path), 'r') as f: backup_report = json.load(f) report = backup_report['item'] meta = backup_report['meta'] item = item_class(meta=base.Meta.from_dict(meta), item=report) message_bytes = item.to_transform_message(logger) message = json.loads(message_bytes) expected_fields = ( 'id', 'type', 'created_at', 'storefront', 'data_source', 'application', 'file_path', ) assert len(expected_fields) == len(message) assert all(f in message for f in expected_fields)