# pylint: disable=redefined-outer-name,unused-argument,invalid-name,too-many-locals,protected-access import json import os from contextlib import nullcontext as does_not_raise from datetime import datetime from os.path import join as p_join from typing import Union import boto3 import pytest from _pytest.fixtures import SubRequest from _pytest.python_api import RaisesContext from freezegun.api import FrozenDateTimeFactory from jsonschema.exceptions import RefResolutionError, ValidationError from mypy_boto3_s3 import S3Client from smelog.factory import SmeBoundLogger from dapd_public_api_scraper import const from dapd_public_api_scraper.config.entity import S3 from dapd_public_api_scraper.const import DSPEnum, EntityEnum from dapd_public_api_scraper.repository.validator import ( NoSchemaLoadedError, SchemaNotFoundError, Validator, ) from tests.utils import get_json_from_path @pytest.fixture(scope='function') def apple_music_get_playlist_response(s3_backup_apple_music_dir): with open( p_join(s3_backup_apple_music_dir, 'playlist/pl.3343f35d630744efa7e01543e415ffeb.json'), 'r' ) as f: yield json.load(f)['item'] def test_not_loaded_validator(logger, apple_music_get_playlist_response): validator = Validator(logger) with pytest.raises(NoSchemaLoadedError): validator.validate(apple_music_get_playlist_response) def test_load_from_fs(logger, apple_music_get_playlist_response, schemas_apple_music_dir): schema_path = p_join(schemas_apple_music_dir, 'playlist', '2020-11-01.json') validator = Validator(logger) validator.load_from_fs(schema_path) validator.validate(apple_music_get_playlist_response) @pytest.fixture(scope='function') def load_apple_music_schema(s3_client, schemas_apple_music_dir): bucket = 'test-delphi-config' local_file_path = p_join(schemas_apple_music_dir, 'playlist/2020-11-01.json') s3_client.create_bucket(Bucket=bucket) s3_client.upload_file( Filename=local_file_path, Bucket=bucket, Key='public-data/schemas/raw/apple_music/playlist/2019-11-01.json' ) s3_client.upload_file( Filename=local_file_path, Bucket=bucket, Key='public-data/schemas/raw/apple_music/playlist/2020-05-01.json' ) s3_client.upload_file( Filename=local_file_path, Bucket=bucket, Key='public-data/schemas/raw/apple_music/playlist/2020-11-01.json' ) yield bucket @pytest.fixture def load_apple_music_schemas__indirect( s3_client: S3Client, schemas_apple_music_dir: str, request: SubRequest, ) -> str: bucket = 'test-delphi-config' s3_client.create_bucket(Bucket=bucket) local_file_path = p_join(schemas_apple_music_dir, 'playlist/2020-11-01.json') for schema_name in request.param: s3_client.upload_file( Filename=local_file_path, Bucket=bucket, Key=f'public-data/schemas/raw/apple_music/playlist/{schema_name}' ) yield bucket @pytest.mark.parametrize( 'load_apple_music_schemas__indirect', [('2018-05-01', '2019-06-01', '2021-11-01')], indirect=['load_apple_music_schemas__indirect'], ) @pytest.mark.parametrize( 'today, expected, expected_exception', [ ('2018-04-01', 'error', pytest.raises(SchemaNotFoundError)), ('2018-05-05', '2018-05-01.json', does_not_raise()), ('2019-06-06', '2019-06-01.json', does_not_raise()), ('2021-10-01', '2019-06-01.json', does_not_raise()), ('2021-11-11', '2021-11-01.json', does_not_raise()), ] ) def test__find_schema( s3_client: S3Client, freezer: FrozenDateTimeFactory, load_apple_music_schemas__indirect: str, today: str, expected: str, expected_exception: Union[does_not_raise, RaisesContext], ): freezer.move_to(today) bucket = load_apple_music_schemas__indirect with expected_exception: result = Validator._find_schema( s3_client=s3_client, bucket=bucket, prefix='public-data/schemas/raw', dsp=DSPEnum.apple_music, entity=EntityEnum.playlist, today=datetime.today(), ) result_schema_name = os.path.split(result)[-1] assert result_schema_name == expected @pytest.mark.parametrize( 'today', [ ('2019-11-01'), ('2019-11-02'), ('2020-05-01'), ('2020-05-02'), ('2020-11-01'), ('2020-11-11'), ] ) @pytest.mark.freeze_time def test_load_from_s3( logger, freezer, apple_music_get_playlist_response, load_apple_music_schema, today ): freezer.move_to(today) bucket = load_apple_music_schema s3_client = boto3.client('s3') validator = Validator(logger) validator.load_from_s3( s3_client, S3(name=p_join(bucket, 'public-data/schemas/raw')), const.DSPEnum.apple_music, const.EntityEnum.playlist, datetime.today() ) validator.validate(apple_music_get_playlist_response) @pytest.mark.parametrize( 'today,expected', [ ('2019-10-01', SchemaNotFoundError), ('2015-10-01', SchemaNotFoundError), ('2019-10-31', SchemaNotFoundError), ] ) @pytest.mark.freeze_time def test_load_from_s3_missing_schema( logger, freezer, apple_music_get_playlist_response, load_apple_music_schema, today, expected ): freezer.move_to(today) bucket = load_apple_music_schema s3_client = boto3.client('s3') validator = Validator(logger) with pytest.raises(expected): validator.load_from_s3( s3_client, S3(name=p_join(bucket, 'public-data/schemas/raw')), const.DSPEnum.apple_music, const.EntityEnum.playlist, datetime.today() ) validator.validate(apple_music_get_playlist_response) def test_validator(logger, apple_music_get_playlist_response, schemas_apple_music_dir): data = apple_music_get_playlist_response validator = Validator(logger) validator.load_from_fs( f'{schemas_apple_music_dir}/{const.EntityEnum.playlist.value}/2020-11-01.json' ) validator.validate(data) @pytest.mark.parametrize( 'entity, file_name, expectation', [ ('album', '1031998253_valid.json', does_not_raise()), ('artist', '129045039_valid.json', does_not_raise()), ('playlist', 'pl.3343f35d630744efa7e01543e415ffeb_valid.json', does_not_raise()), ('track', '1126431409_valid.json', does_not_raise()), ('album', '1031998253_invalid.json', pytest.raises(ValidationError)), ('artist', '129045039_invalid.json', pytest.raises(ValidationError)), ( 'playlist', 'pl.3343f35d630744efa7e01543e415ffeb_invalid.json', pytest.raises(ValidationError) ), ('track', '1126431409_invalid.json', pytest.raises(ValidationError)), ] ) def test_validator__apple_music( logger: SmeBoundLogger, schemas_apple_music_dir: str, raw_data_apple_music_dir: str, entity: str, file_name: str, expectation: Union[does_not_raise, RaisesContext], ): validator = Validator(logger) validator.load_from_fs(p_join(schemas_apple_music_dir, entity, '2021-12-13.json')) data = get_json_from_path(p_join(raw_data_apple_music_dir, entity, file_name)) with expectation: validator.validate(data) @pytest.mark.parametrize( 'entity, file_name, expectation', [ ('album', '0GHyMOR7XVVIU0NwQSrw90_valid.json', does_not_raise()), ('artist', '0cQuYRSzlItquYxsQKDvVc_valid.json', does_not_raise()), ('playlist', '1T0WUWrDqIGt7OvHNA0amF_valid.json', does_not_raise()), ('track', '0E8qIw6KblOtDQYzzHQrxL_valid.json', does_not_raise()), ('album', '0GHyMOR7XVVIU0NwQSrw90_invalid.json', pytest.raises(ValidationError)), ('artist', '0cQuYRSzlItquYxsQKDvVc_invalid.json', pytest.raises(ValidationError)), ('playlist', '1T0WUWrDqIGt7OvHNA0amF_invalid.json', pytest.raises(ValidationError)), ('track', '0E8qIw6KblOtDQYzzHQrxL_invalid.json', pytest.raises(ValidationError)), ] ) def test_validator__spotify( logger: SmeBoundLogger, schemas_spotify_dir: str, raw_data_spotify_dir: str, entity: str, file_name: str, expectation: Union[does_not_raise, RaisesContext], ): validator = Validator(logger) validator.load_from_fs(p_join(schemas_spotify_dir, entity, '2021-12-13.json')) data = get_json_from_path(p_join(raw_data_spotify_dir, entity, file_name)) with expectation: validator.validate(data) def test_validator_extra_field(logger, apple_music_get_playlist_response, schemas_apple_music_dir): data = apple_music_get_playlist_response data['data'][0]['id2'] = 'Extra field' validator = Validator(logger) validator.load_from_fs( f'{schemas_apple_music_dir}/{const.EntityEnum.playlist.value}/2020-11-01.json' ) validator.validate(data) def test_validator_invalid_schema( logger, apple_music_get_playlist_response, schemas_apple_music_dir ): data = apple_music_get_playlist_response validator = Validator(logger) with pytest.raises(RefResolutionError): validator.load_from_fs( f'{schemas_apple_music_dir}/{const.EntityEnum.playlist.value}/2020-11-01_invalid.json' ) validator.validate(data) def test_validator_validation_error( logger, apple_music_get_playlist_response, schemas_apple_music_dir ): data = apple_music_get_playlist_response data['data'][0]['id'] = 1234 validator = Validator(logger) with pytest.raises(ValidationError): validator.load_from_fs( f'{schemas_apple_music_dir}/{const.EntityEnum.playlist.value}/2020-11-01.json' ) validator.validate(data) def test_validator_missing_required_error( logger, apple_music_get_playlist_response, schemas_apple_music_dir ): data = apple_music_get_playlist_response del data['data'][0]['id'] validator = Validator(logger) with pytest.raises(ValidationError): validator.load_from_fs( f'{schemas_apple_music_dir}/{const.EntityEnum.playlist.value}/2020-11-01.json' ) validator.validate(data)