import gzip import io import json import unittest import unittest.mock from os.path import abspath, dirname import boto3 import pytest from moto import mock_s3 import slz_downloader.dsp.apple import slz_downloader.entities from slz_downloader.dsp.apple import unzip, validate_apple_zipped_reports PATH_TO_APPLE_FIXTURES = dirname(dirname(abspath(__file__))) + '/fixtures/apple/' @pytest.fixture def apple_job(): return slz_downloader.entities.Job( uow_id='apple-20210211-sme-amStreams-v1_2', unit_of_work_id=0, dsp='apple', report_type='amStreams', subtype='Detailed', version='v1_2', report_date='2021-02-11', licensor='sme', extension='txt', config_bucket='delphi-configs', context='4632::87165506', context_params=None, job_id="test_job_id", ) @pytest.fixture def apple_job_gz(): return slz_downloader.entities.Job( uow_id='apple-20210211-sme-amContent-v1_1', unit_of_work_id=0, dsp='apple', report_type='amContent', subtype='Detailed', version='v1_1', report_date='2024-09-11', licensor='sme', extension='txt', config_bucket='delphi-configs', context='4632::87165506', context_params=None, job_id="test_job_id", ) @unittest.mock.patch('slz_downloader.dsp.apple.get_secret') @unittest.mock.patch('requests.post') @mock_s3 def test_downloading_zip(requests_post, get_secret, params, apple_job): boto3.setup_default_session() conn = boto3.resource('s3', region_name='us-east-1') for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']: conn.create_bucket(Bucket=bucket) # Client initialization logger = unittest.mock.Mock() client = slz_downloader.dsp.apple.Client(logger=logger) # Mock secrets get_secret.return_value = json.dumps({ 'user_id': '123', 'access_token': '234', }) client.configure(params) zip_file_name = 'AppleMusic_Streams_80031942_20190901_V1_2 .zip' with open(PATH_TO_APPLE_FIXTURES + zip_file_name, 'rb') as f: # Mock response from source API resp_mock = unittest.mock.MagicMock() requests_post.return_value.__enter__.return_value = resp_mock resp_mock.ok = True resp_mock.url = 'source_url' resp_mock.raw = f resp_mock.headers = {'Content-Length': 123} client.download(apple_job, chunk_size=1) # check API called with correct request requests_post.assert_called_with( 'https://reportingitc-reporter.apple.com/reportservice/sales/v1', data={ 'jsonRequest': json.dumps( { 'userid': '123', 'accesstoken': '234', 'version': '1.0', 'mode': 'Robot.XML', 'account': '4632', 'queryInput': '[p=Reporter.properties, Sales.getReport, 87165506,amStreams,' 'Detailed,Daily,20210211,1_2]' } ), }, headers={ 'content-type': 'application/x-www-form-urlencoded', }, stream=True, timeout=50, ) # read data from mocked S3 buckets path = 'apple/streams/v1_2/report_date=2021-02-11/report_licensor=sme' actual_compressed = conn.Object( 'bucket-archive-quarantine', f'{path}/AppleMusic_Streams_87165506_20210211_V1_2.zip' ).get()['Body'].read() with open(PATH_TO_APPLE_FIXTURES + zip_file_name, 'rb') as f: test_file_content = f.read() assert test_file_content == actual_compressed @unittest.mock.patch('slz_downloader.dsp.apple.get_secret') @unittest.mock.patch('requests.post') @mock_s3 def test_downloading_gz(requests_post, get_secret, params, apple_job_gz): boto3.setup_default_session() conn = boto3.resource('s3', region_name='us-east-1') for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']: conn.create_bucket(Bucket=bucket) # Client initialization logger = unittest.mock.Mock() client = slz_downloader.dsp.apple.Client(logger=logger) # Mock secrets get_secret.return_value = json.dumps({ 'user_id': '123', 'access_token': '234', }) client.configure(params) expected = 'Some bytes data' data = bytes(expected, 'utf-8') expected_compressed = gzip.compress(data, 9) # Mock response from source API resp_mock = unittest.mock.MagicMock() requests_post.return_value.__enter__.return_value = resp_mock resp_mock.ok = True resp_mock.url = 'source_url' resp_mock.raw = io.BytesIO(expected_compressed) resp_mock.headers = {'Content-Length': 123} client.download(apple_job_gz, chunk_size=1) # check API called with correct request requests_post.assert_called_with( 'https://reportingitc-reporter.apple.com/reportservice/sales/v1', data={ 'jsonRequest': json.dumps( { 'userid': '123', 'accesstoken': '234', 'version': '1.0', 'mode': 'Robot.XML', 'account': '4632', 'queryInput': '[p=Reporter.properties, Sales.getReport, 87165506,amContent,' 'Detailed,Daily,20240911,1_1]' } ), }, headers={ 'content-type': 'application/x-www-form-urlencoded', }, stream=True, timeout=50, ) # read data from mocked S3 buckets path = 'apple/content/v1_1/report_date=2024-09-11/report_licensor=sme' actual_compressed = conn.Object( 'bucket-archive-quarantine', f'{path}/AppleMusic_Content_87165506_20240911_V1_1.txt.gz' ).get()['Body'].read() actual_decompressed = conn.Object( 'bucket-decompressed-quarantine', f'{path}/AppleMusic_Content_87165506_20240911_V1_1.txt' ).get()['Body'].read() assert expected == gzip.decompress(actual_compressed).decode('utf8') assert expected == actual_decompressed.decode('utf8') @pytest.mark.parametrize( 'input, expected_result', [ (['test_chunk_[1-2].txt.gz', 'test_chunk_[2-2].txt.gz'], True), (['test_chunk_[1-1].txt.gz'], True), (['test_chunk.txt.gz'], True), ( ['test_chunk_[1-11].txt.gz', 'test_chunk_[10-11].txt.gz', 'test_chunk_[11-11].txt.gz' ], False ), (['test_chunk_[1-2].txt.gz', 'test_chunk_[3-2].txt.gz'], False), (['test_chunk_[1-2].txt.gz', 'test_chunk_[2-3].txt.gz'], False), (['test_chunk.txt.gz', 'test_chunk_[1-1].txt.gz'], False), ([], False), ] ) def test_zip_report_validatition(input, expected_result): result = validate_apple_zipped_reports(input) assert result == expected_result def test_unzip_file_brackets(): zip_file_name = dirname( dirname(abspath(__file__)) ) + "/fixtures/apple/AppleMusic_Streams_80031942_20190901_V1_2 .zip" for name, stream in unzip(zip_file_name): assert '(' in name assert '[' not in name