import gzip import json import unittest.mock from datetime import datetime from unittest.mock import call from requests.models import Response from slz_downloader.dsp.appreciationengine.clients import MembersVisitsTotalClient from slz_downloader.entities import Job def resp_mock(): data = [ # call to segments [ { 'ID': '1701', 'BrandName': 'Starfleet', 'URL': 'test', }, { 'ID': '23456', 'BrandName': 'Enterprise', 'URL': 'test', }, { 'ID': '234', 'BrandName': 'Enterprise', 'URL': None, } ], { 'metrics': { 'visits': 76611, 'registrations': 19838, 'conversion': 26, } }, { 'metrics': { 'visits': 11, 'registrations': 38, 'conversion': 0, } } ] for dt in data: resp_mock = unittest.mock.MagicMock(spec=Response) resp_mock.ok = True resp_mock.url = '' resp_mock.content = bytes(json.dumps(dt), encoding='utf8') resp_mock.headers = {} yield resp_mock @unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret') @unittest.mock.patch('requests.get') @unittest.mock.patch( 'slz_downloader.dsp.appreciationengine.clients.datetime', wraps=datetime, now=lambda *args, **kwargs: datetime(year=2021, month=6, day=5, hour=1, minute=1, second=1) ) def test_downloading_membersvisitstotal( dt_mock, requests_get, get_secret, params, s3_client, ae_mapping ): for bucket in [ 'bucket-archive-quarantine', 'bucket-decompressed-quarantine', 'bucket-decompressed' ]: s3_client.create_bucket(Bucket=bucket) uow_dict = { 'uow_id': 'appreciationengine-20210415-sme-membersvisittotals-v1', 'dsp': 'appreciationengine', 'report_type': 'membersvisittotals', 'version': 'v1', 'report_date': '2021-04-15', 'licensor': 'sme', 'extension': 'json', 'context': 'CenturyMediaRecords', } # Client initialization logger = unittest.mock.Mock() client = MembersVisitsTotalClient(logger=logger) # Mock secrets get_secret.return_value = json.dumps({ 'Sony Music - Century Media Records': 'token72739d3', }) client.configure(params) job = Job.from_dict(uow_dict) # Mock response from source API expected_file_content = \ '{"metrics": {"visits": 76611, "registrations": 19838, "conversion": 26}, "ID": "1701"}\n' \ '{"metrics": {"visits": 11, "registrations": 38, "conversion": 0}, "ID": "23456"}\n' requests_get.return_value.__enter__.side_effect = resp_mock() url = 'https://sme.theappreciationengine.com/v1.1/members/visit/totals' client.download(job, chunk_size=1) # Appreciation Engine API called with correct request requests_get.assert_has_calls( [ call( 'https://sme.theappreciationengine.com/v1.1/segments', { 'apiKey': 'token72739d3', 'excludeBasicSocial': 1, 'withActivities': 1, } ), call(url, { 'apiKey': 'token72739d3', 'overall': True, 'segment': '1701' }), call(url, { 'apiKey': 'token72739d3', 'overall': True, 'segment': '23456' }), ], any_order=True ) # read data from mocked S3 buckets path = 'appreciationengine/membersvisittotals/v1/report_date=2021-04-15/report_licensor=sme' actual_decompressed = s3_client.get_object( Bucket='bucket-decompressed-quarantine', Key=f'{path}/CenturyMediaRecords_20210415_20210605010101.json', )['Body'].read() actual_compressed = s3_client.get_object( Bucket='bucket-archive-quarantine', Key=f'{path}/CenturyMediaRecords_20210415_20210605010101.json.gz', )['Body'].read() assert expected_file_content == actual_decompressed.decode('utf8') assert expected_file_content == gzip.decompress(actual_compressed).decode('utf8') @unittest.mock.patch('slz_downloader.dsp.appreciationengine.clients.get_secret') @unittest.mock.patch('requests.get') def test_ae_availability_check_membersvisitstotal(requests_get, get_secret, params, ae_mapping): uow = { 'uow_id': 'appreciationengine-20210415-sme-membersvisittotals-v1', 'dsp': 'appreciationengine', 'report_type': 'membersvisittotals', 'version': 'v1', 'report_date': '2021-04-15', 'licensor': 'sme', 'extension': 'json', 'context': 'CenturyMediaRecords', } job = Job.from_dict(uow) logger = unittest.mock.Mock() requests_get.return_value.__enter__.side_effect = resp_mock() # Mock secrets get_secret.return_value = json.dumps({ 'Sony Music - Century Media Records': 'token72739d3', }) client = MembersVisitsTotalClient(logger) client.configure(params) result = client._check_source_is_available(job) assert result == ('CenturyMediaRecords', 0)