import logging from unittest import TestCase, mock import pytest from db_schema.postgres.connection import Connection from smelog.entities import LoggerConfig from smelog.factory import LoggerFactory from slz_downloader import manager from slz_downloader.config import App from .conftest import creds_loader class Worker: def __init__(self, db_conn, *args, **kwargs): self.db_conn = db_conn def process(self, job): return True def initializer(*args, **kwargs): return Worker(db_conn=mock.Mock()) def initializer_db_conn(*args, **kwargs): conn = Connection(credentials_loader=creds_loader, name='s3_storage_test', version='v1') conn.session.execute('SELECT 1') return Worker(db_conn=conn) class ManagerTestCase(TestCase): def setUp(self): logger_name = f'logger_{self._testMethodName}' self._log_config = LoggerConfig( name=logger_name, version='1', level=logging.DEBUG, environment='test', is_local=False, is_multiprocessing=True, ) self._logger = LoggerFactory(self._log_config).get_logger(logger_name) self._params = App.empty() self._uow = { "dsp": "spotify", "extension": "json", "uow_id": "spotify-20200110-smejp-aggregatedstreams-v2", "licensor": "smejp", "config_bucket": "qa-delphi-configs", "report_type": "aggregatedstreams", "contexts": ["US", "RU", "GB"], "priority": 5, "version": "v2", "subtype": None, "config_key": "batch-run-config.json", "job_id": "1637_2020-01-11T21.31.58", "report_date": "2020-01-10", } self._dsp_settings = mock.MagicMock() self._dsp_settings.__reduce__ = lambda self: (mock.MagicMock, ()) self._dsp_criterias = mock.MagicMock() self._dsp_criterias.__reduce__ = lambda self: (mock.MagicMock, ()) def tearDown(self) -> None: self._logger.close() def test_single_process(self): self._params.downloader.download_concurrency = 1 mgr = manager.Manager( self._logger, self._params, self._uow, self._dsp_settings, self._dsp_criterias, initializer, False, ) contexts = ['ax', 'bx', 'cx'] result = mgr.run(contexts) self.assertDictEqual(result, {'ax': True, 'bx': True, 'cx': True}) def test_pool(self): self._params.downloader.download_concurrency = 5 mgr = manager.Manager( self._logger, self._params, self._uow, self._dsp_settings, self._dsp_criterias, initializer, False, ) contexts = ['ax', 'bx', 'cx'] self._dsp_criterias.get_criteria_for_context.side_effect = [{}, {}, {}] result = mgr.run(contexts, 30) self.assertDictEqual(result, {'ax': True, 'bx': True, 'cx': True}) @pytest.mark.skip(reason="works only locally, investigate") def test_connections(self): self._params.downloader.download_concurrency = 2 mgr = manager.Manager( self._logger, self._params, self._uow, self._dsp_settings, self._dsp_criterias, initializer_db_conn, False, ) contexts = ['ax', 'bx', 'cx'] self._dsp_criterias.get_criteria_for_context.side_effect = [{}, {}, {}] result = mgr.run(contexts, 30) self.assertDictEqual(result, {'ax': True, 'bx': True, 'cx': True})