from os import path import pytest from sample_validator import load_schema, validate_stream FIXTURES_PATH = path.join(path.dirname(__file__), "fixtures") @pytest.mark.parametrize('description,report_url,schema_url,rows_number,rate,expected_result,expected_log', ( ( 'aggregatedstreams', 'spotify/aggregatedstreams/spotify_2017-09-25_sony_v2.0_aggregatedstreams_100.ndjson', 'spotify/aggregatedstreams/v2/2017-09-04.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'aggregatedstreams empty', 'spotify/aggregatedstreams/spotify_2017-09-25_sony_v2.0_aggregatedstreams_0.ndjson', 'spotify/aggregatedstreams/v2/2017-09-04.json', 100, 0, False, { 'errors_len': 1, 'warnings_len': 0, } ), ( 'aggregatedstreams spoiled', 'spotify/aggregatedstreams/spotify_2017-09-25_sony_v2.0_aggregatedstreams_100_spoiled.ndjson', 'spotify/aggregatedstreams/v2/2017-09-04.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 1, } ), ( 'streams v1', 'spotify/streams/spotify_2014-01-01_sony_v1.0_ad_streams_100.ndjson', 'spotify/streams/v1/2014-01-01.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'streams v2', 'spotify/streams/spotify_2019-06-23_smej_v2.2_im_streams.ndjson', 'spotify/streams/v2/2017-09-04.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'streams v2 spoiled', 'spotify/streams/spotify_2014-01-01_sony_v1.0_ad_streams_100_spoiled.ndjson', 'spotify/streams/v1/2014-01-01.json', 100, 0, False, { 'errors_len': 1, 'warnings_len': 0, } ), ( 'streams v1 spoiled', 'spotify/streams/spotify_2014-01-01_sony_v1.0_ad_streams_100_spoiled.ndjson', 'spotify/streams/v1/2014-01-01.json', 100, 0, False, { 'errors_len': 1, 'warnings_len': 0, } ), ( 'streams v1 spoiled, required field is missing but number of failed rows > 1: file is valid, errors were in log_info', 'spotify/streams/spotify_2014-01-01_sony_v1.0_ad_streams_100_spoiled.ndjson', 'spotify/streams/v1/2014-01-01.json', 100, 0.5, True, { 'errors_len': 1, 'warnings_len': 0, } ), ( 'sub_30_sec_streams', 'spotify/sub_30_sec_streams/spotify_2018-01-01_smej_v2.4_cz_sub_30_sec_streams_100.ndjson', 'spotify/sub_30_sec_streams/v2/2018-01-01.json', 100, 0.0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'sub_30_sec_streams, many rows', 'spotify/sub_30_sec_streams/spotify_2018-01-01_smej_v2.4_cz_sub_30_sec_streams_100.ndjson', 'spotify/sub_30_sec_streams/v2/2018-01-01.json', 100000, 0.0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'tracks', 'spotify/tracks/spotify_2014-01-01_sony_v1.0_tracks_100.ndjson', 'spotify/tracks/v1/2014-01-01.json', 100, 0.0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'users v1', 'spotify/users/spotify_2014-01-01_sony_v1.0_users_100.ndjson', 'spotify/users/v1/2014-01-01.json', 100, 0.0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'users v2', 'spotify/users/spotify_2018-03-12_smejintl_v2.0_users_100.ndjson', 'spotify/users/v2/2017-09-04.json', 100, 0.0, True, { 'errors_len': 0, 'warnings_len': 0, } ), )) def test_spotify( description, s3_upload_schema, s3_upload_report, report_url, schema_url, rows_number, rate, expected_result, expected_log, ): report_url = s3_upload_report(report_url) schema_url = s3_upload_schema(schema_url) result, log_info = validate_stream( load_schema(schema_url), report_url, rows_number, rate, ) assert result == expected_result assert len(log_info['errors']) == expected_log['errors_len'] assert len(log_info['warnings']) == expected_log['warnings_len']