# pylint: disable=redefined-outer-name,too-many-arguments,fixme import json from dataclasses import dataclass from datetime import datetime from os.path import join as p_join from typing import Optional, Type from unittest import mock import pytest from _pytest.fixtures import SubRequest from freezegun.api import FrozenDateTimeFactory from mypy_boto3_s3 import S3Client from dapd_public_api_scraper.config.entity import StorageConfig from dapd_public_api_scraper.entity import apple_music as apple_music_entity from dapd_public_api_scraper.entity import base from dapd_public_api_scraper.entity import spotify as spotify_entity from dapd_public_api_scraper.service.storage import Storage @dataclass class ItemData: filename: str data_source: str type: str storefront: str @pytest.fixture def init_bucket__indirect_name(s3_client: S3Client, request: SubRequest) -> str: bucket_name = request.param s3_client.create_bucket(Bucket=bucket_name) return bucket_name @pytest.fixture def storage_config() -> StorageConfig: return StorageConfig( raw_data_bucket='raw_data_bucket', corrupted_bucket='corrupted_bucket', ) @pytest.mark.parametrize( 'item', [ spotify_entity.Album, spotify_entity.Artist, spotify_entity.Playlist, spotify_entity.Track, apple_music_entity.Album, apple_music_entity.Artist, apple_music_entity.Playlist, apple_music_entity.Track, ] ) @pytest.mark.parametrize( 'bucket, filename, extension, item_data, timestamp_now, freeze_now, expected', [ ( 'bucket', None, None, ItemData(filename='name1', data_source='one', type='two', storefront='three'), datetime(year=2020, month=1, day=2, hour=11), None, 's3://bucket/ds=one/entity=two/storefront=three/y=2020/m=1/d=2/h=11/name1.json', ), ( 'bucket0', 'name', None, ItemData(filename='name2', data_source='blah', type='bleh', storefront='meh'), datetime(year=2021, month=2, day=3, hour=12), None, 's3://bucket0/ds=blah/entity=bleh/storefront=meh/y=2021/m=2/d=3/h=12/name.json', ), ( 'bucket0', None, 'bat', ItemData(filename='name3', data_source='blah', type='bleh', storefront='meh'), datetime(year=2021, month=2, day=3, hour=12), None, 's3://bucket0/ds=blah/entity=bleh/storefront=meh/y=2021/m=2/d=3/h=12/name3.bat', ), ( 'bucket2', 'nameA', 'txt', ItemData(filename='name4', data_source='spotify', type='track', storefront='global'), datetime(year=2022, month=3, day=15, hour=6), None, 's3://bucket2/ds=spotify/entity=track/storefront=global/y=2022/m=3/d=15/h=6/nameA.txt', ), ( 'bucket3', 'nameB', 'txt', ItemData(filename='name5', data_source='spotify', type='album', storefront='global'), None, datetime(year=2018, month=3, day=18, hour=12), 's3://bucket3/ds=spotify/entity=album/storefront=global/y=2018/m=3/d=18/h=12/nameB.txt', ), ] ) def test_get_file_path_for_item( storage_config: StorageConfig, freezer: FrozenDateTimeFactory, item: Type[base.Item], bucket: str, item_data: ItemData, filename: Optional[str], extension: str, timestamp_now: Optional[datetime], freeze_now: Optional[datetime], expected: str, ): # TODO: it doesn't matter what type of item is used. Only what's written into meta makes the # the difference. Might be a good place to improve - let Item populate its meta data # and don't allow to overwrite those values. item = item( item={ 'id': item_data.filename, 'data': [{ 'id': item_data.filename }], # Mimic filename source for known vendors. 'item': { 'id': item_data.filename }, }, meta=base.Meta( item_id='123', item_type=item_data.type, item_ts='2022-01-25T19:35:31.013492', item_storefront=item_data.storefront, data_source=item_data.data_source, app_version='0.0.1', ), ) if freeze_now: freezer.move_to(freeze_now) storage = Storage( logger=mock.Mock(), config=storage_config, ) result = storage.get_file_path_for_item( bucket_name=bucket, item=item, timestamp=timestamp_now, filename=filename, extension=extension, ) assert result == expected @pytest.mark.parametrize( 'init_bucket__indirect_name, relative_path, content', [ ('bbb', 'b.json', b'a'), ('ccc', 'c.json', json.dumps({ 'test': 'a', 'f': 123, 'array': [] }).encode('utf-8')), ('ddd', 'd/e.json', b'abc'), ('eee', 'f/g/h.json', b'abc'), ], indirect=['init_bucket__indirect_name'] ) def test_upload_bytes( s3_client: S3Client, storage_config: StorageConfig, init_bucket__indirect_name: str, relative_path: str, content: bytes, ): bucket_name = init_bucket__indirect_name file_path = p_join('s3://', bucket_name, relative_path) # initially no files in bucket assert 'Contents' not in s3_client.list_objects(Bucket=bucket_name) storage = Storage( logger=mock.Mock(), config=storage_config, ) is_success = storage.upload_bytes(file_path=file_path, file=content) assert is_success bucket_content = s3_client.get_object(Bucket=bucket_name, Key=relative_path) assert bucket_content['Body'].read() == content files_after_upload = s3_client.list_objects(Bucket=bucket_name)['Contents'] assert len(files_after_upload) == 1 assert files_after_upload[0]['Key'] == relative_path @pytest.mark.parametrize( 'file_path, content', [ ('./local/invalid/path/a.json', b'asd'), ('./a.json', b''), # Empty content shouldn't be uploaded ('s3://invalid_bucket/a.json', b'123') ] ) def test_upload_bytes__fail( s3_client: S3Client, # pylint: disable=unused-argument storage_config: StorageConfig, file_path: str, content: bytes, ): storage = Storage( logger=mock.Mock(), config=storage_config, ) is_success = storage.upload_bytes(file_path, content) assert not is_success