import gzip import json import unittest.mock import pytest from slz_appreciationengine_scrapper.content_status_service import ContentStatusService from slz_appreciationengine_scrapper.dsp.appreciationengine.clients import BusinessUnitClient from slz_appreciationengine_scrapper.entities import Job @pytest.mark.integration @unittest.mock.patch('slz_appreciationengine_scrapper.dsp.appreciationengine.clients.get_secret') def test_downloading(get_secret, db, params, ae_mapping, s3_client): for bucket in ['bucket-archive-quarantine', 'bucket-decompressed-quarantine']: s3_client.create_bucket(Bucket=bucket) uow_dict = { 'uow_id': 'appreciationengine-20201104-sme-businessunit-v1', 'dsp': 'appreciationengine', 'report_type': 'businessunit', 'version': 'v1', 'report_date': '2020-11-04', 'licensor': 'sme', 'extension': 'json', 'context': 'businessunit', } get_secret.return_value = json.dumps({}) # Client initialization logger = unittest.mock.Mock() client = BusinessUnitClient(logger=logger) client.configure(params) job = Job.from_dict(uow_dict) content_status_service = ContentStatusService(logger=logger, db_conn=db) client.download(job, chunk_size=1, content_status_service=content_status_service) # read data from mocked S3 buckets path = 'appreciationengine/businessunit/v1/report_date=2020-11-04/report_licensor=sme' actual_decompressed = s3_client.get_object( Bucket='bucket-decompressed-quarantine', Key=f'{path}/businessunit_20201104.json', )['Body'].read() actual_compressed = s3_client.get_object( Bucket='bucket-archive-quarantine', Key=f'{path}/businessunit_20201104.json.gz', )['Body'].read() expected_part = [ { 'bu_name': 'CenturyMediaRecords', 'bu_title': 'Sony Music - Century Media Records', }, { 'bu_name': 'AU_NZ', 'bu_title': 'Sony Music Australia/New Zealand', } ] assert all(x in json.loads(actual_decompressed.decode('utf8')) for x in expected_part) assert all( x in json.loads(gzip.decompress(actual_compressed).decode('utf8')) for x in expected_part )