import json from unittest import mock from unittest.mock import patch import pytest from requests import Response from slz_appreciationengine_scrapper.config import Scrapper from slz_appreciationengine_scrapper.const import RATE_LIMIT_MINUTE_THRESHOLD from slz_appreciationengine_scrapper.dsp.appreciationengine.ae_io import AEBytesIO from slz_appreciationengine_scrapper.entities import Job class SmartOpen: def __enter__(self): return self def __exit__(self, exc_type, exc_val, exc_tb): return @staticmethod def read(): return '{"sme": {"US": "mapping_key"}}' @pytest.fixture(scope='function') def ae_client(): logger = mock.Mock() job = Job.empty() job.report_type = 'activityfeed' job.licensor = 'sme' job.context = 'US' with patch('smart_open.open', return_value=SmartOpen()): client = AEBytesIO( logger=logger, job=job, secret={'mapping_key': 'ApiKey'}, scrapper_config=Scrapper(host='', ), config_bucket='test-config-bucket', troubleshooting_logs_bucket='', ) yield client def test_after_request_actions(ae_client: AEBytesIO) -> None: assert ae_client._meta.get('request_delay') is None resp_mock_base = mock.MagicMock(spec=Response) resp_mock_base.headers = {RATE_LIMIT_MINUTE_THRESHOLD: 60} ae_client._after_request_actions(response=resp_mock_base) assert ae_client._meta.get('request_delay') == 1.0 @mock.patch('requests.get') @mock.patch('time.sleep') def test_slow_down_requests(sleep_mock, requests_get, ae_client) -> None: resp_mock_base = mock.MagicMock(spec=Response) resp_mock_base.ok = True dt = {'items': [{'ID': '123'}], 'totalSize': 1} resp_mock_base.content = bytes(json.dumps(dt), encoding='utf8') resp_mock_base.headers = {RATE_LIMIT_MINUTE_THRESHOLD: 60} resp_mock_base.url = '' resp_mock_base.status_code = 200 resp_mock_base.reason = 'Some text' requests_get.return_value = resp_mock_base requests_get.return_value.__enter__.return_value = resp_mock_base ae_client.read() assert ae_client._meta.get('request_delay') == 1.0 ae_client._stream = None resp_mock_base.headers = {RATE_LIMIT_MINUTE_THRESHOLD: 10} ae_client.read() assert ae_client._meta.get('request_delay') == 6.0 assert sleep_mock.call_count == 1