import pytest from unittest import mock from sample_validator import load_schema, validate_stream @pytest.mark.parametrize('description,report_url,schema_url,rows_number,rate,expected_result,expected_log', ( ( 'unprintable symbols', 'apple/content/AppleMusic_Content_80031742_20190903_V1_2_short.tsv', 'apple/content/v1_2/2019-05-13.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'demographics v1_0', 'apple/artist_demographics/apple_2018-01-01_smej_86875390_v1_0_amArtistDemographics_100.tsv', 'apple/artist_demographics/v1_0/2017-08-02.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'demographics v1_0 empty', 'apple/artist_demographics/apple_2018-01-01_smej_86875390_v1_0_amArtistDemographics_0.tsv', 'apple/artist_demographics/v1_0/2017-08-02.json', 100, 0, False, { 'errors_len': 1, 'warnings_len': 0, } ), ( 'demographics v1_0 spoiled datestamps', 'apple/artist_demographics/apple_2018-01-01_smej_86875390_v1_0_amArtistDemographics_100_spoiled.tsv', 'apple/artist_demographics/v1_0/2017-08-02.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 1, } ), ( 'demographics v1_1', 'apple/artist_demographics/apple_2019-01-26_sony_80031738_v1_1_amArtistDemographics_100.tsv', 'apple/artist_demographics/v1_1/2018-12-13.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 100, } ), ( 'content v1_0, schema date == report date', 'apple/content/apple_2016-04-27_sony_86759336_v1_0_amContent_100.tsv', 'apple/content/v1_0/2016-04-27.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'content v1_0, schema date < report date ', 'apple/content/apple_2017-08-02_theorchard_80032084_v1_0_amContent_100.tsv', 'apple/content/v1_0/2017-07-20.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'content v1_1, schema date < report date ', 'apple/content/apple_2018-01-03_theorchard_80035150_v1_1_amContent_100.tsv', 'apple/content/v1_1/2017-11-16.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'content v1_2, schema date < report date', 'apple/content/apple_2019-06-01_sony_85406437_v1_2_amContent_100.tsv', 'apple/content/v1_2/2019-05-13.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'content_demographics v1_0, schema date == report date', 'apple/content_demographics/apple_2017-08-02_sony_86759191_v1_0_amContentDemographics_100.tsv', 'apple/content_demographics/v1_0/2017-08-02.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'content_demographics v1_1, schema date < report date', 'apple/content_demographics/apple_2019-06-01_sony_80031740_v1_1_amContentDemographics_100.tsv', 'apple/content_demographics/v1_1/2018-12-13.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 100, } ), ( 'event v1_0, schema date == report date', 'apple/event/apple_2016-04-28_sony_86759358_v1_0_amEvent_100.tsv', 'apple/event/v1_0/2016-04-28.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'event v1_1, schema date < report date', 'apple/event/apple_2017-11-17_theorchard_85420853_v1_1_amEvent_100.tsv', 'apple/event/v1_1/2017-11-16.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'event v1_1, schema date < report date, many rows', 'apple/event/apple_2017-11-17_theorchard_85420853_v1_1_amEvent_100.tsv', 'apple/event/v1_1/2017-11-16.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'event v1_2, schema date < report date, many rows', 'apple/event/apple_2019-05-16_theorchard_80032084_v1_2_amEvent_100.tsv', 'apple/event/v1_2/2019-05-13.json', 100000, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'non royalty streams v1_0, schema date < report date, many rows', 'apple/non_royalty_streams/apple_2018-02-26_sony_86759255_v1_0_amNonRoyaltyStreams_100.tsv', 'apple/non_royalty_streams/v1_0/2018-02-22.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'non royalty streams v1_1, schema date < report date', 'apple/non_royalty_streams/apple_2019-06-09_sony_80033090_v1_1_amNonRoyaltyStreams_100.tsv', 'apple/non_royalty_streams/v1_1/2019-05-13.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'streams v1_0, schema date < report date', 'apple/streams/apple_2016-04-27_sony_86759336_v1_0_amStreams_100.tsv', 'apple/streams/v1_0/2016-04-27.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'streams v1_0, schema date < report date', 'apple/streams/apple_2017-03-31_theorchard_85420853_v1_0_amStreams_100.tsv', 'apple/streams/v1_0/2017-03-24.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'streams v1_1, schema date < report date', 'apple/streams/apple_2017-11-28_theorchard_80035150_v1_1_amStreams_100.tsv', 'apple/streams/v1_1/2017-11-16.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), ( 'streams v1_2, schema date < report date', 'apple/streams/apple_2019-06-01_theorchard_80031727_v1_2_amStreams_100.tsv', 'apple/streams/v1_2/2019-05-13.json', 100, 0, True, { 'errors_len': 0, 'warnings_len': 0, } ), )) def test_apple_music( description, s3_upload_schema, s3_upload_report, report_url, schema_url, rows_number, rate, expected_result, expected_log, ): report_url = s3_upload_report(report_url) schema_url = s3_upload_schema(schema_url) result, log_info = validate_stream( load_schema(schema_url), report_url, rows_number, rate, ) assert result == expected_result assert len(log_info['errors']) == expected_log['errors_len'] assert len(log_info['warnings']) == expected_log['warnings_len'] @pytest.mark.skip() @pytest.mark.parametrize('description,report_url,schema_url,rows_number,rate,expected_exception', ( ( 'value error', 's3://apple/streams/test_1001.tsv', 'apple/streams/v1_2/2019-05-13.json', 100, 0, ValueError, ), )) def test_correct_raise_exception(description, s3_upload_schema, report_url, schema_url, rows_number, rate, expected_exception): schema_url = s3_upload_schema(schema_url) with mock.patch('tenacity.BaseRetrying.wait', side_effect=lambda *args, **kwargs: 0): with pytest.raises(expected_exception): validate_stream( load_schema(schema_url), report_url, rows_number, rate, )