"""Global test configuration and set of fixtures.""" from unittest.mock import MagicMock from unittest.mock import Mock import garcon import pytest from snowflake import connector from analytics_aggregation import base_config from analytics_aggregation.util import db from analytics_aggregation.util import snowflake_db class SubstringMatcher: """Class which allows to check for SQL calls by the list of substrings.""" def __init__(self, containing): """Initialise object. Args: containing (list): A list of substrings to check. """ self.containing = [el.lower() for el in containing] def __eq__(self, sql): """Equal magic method. Check if all the substrings from the passed list are present in SQL. Args: sql (str): SQL statement from a call. """ sql = sql.lower().replace('\n', ' ') return all(el in sql for el in self.containing) def __repr__(self): """Represent string magic method to play nice with py.test messages.""" return 'SQL containing: {}'.format(', '.join(self.containing)) @pytest.fixture(scope='function') def query_mock(monkeypatch): """Return mock of util.db.Query class.""" query = MagicMock(name='query_mock') query.fill = Mock(return_value=query) query.fill_escaped = Mock(return_value=query) query_mock = MagicMock(return_value=query) monkeypatch.setattr(db, 'Query', query_mock) return query @pytest.fixture(autouse=True) def mock_requests(monkeypatch): """Automatically mock requests.sessions.Session. We do not want tests to perform any real HTTP requests """ session_mock = MagicMock(name='Session') session_mock().__enter__().request.return_value = session_mock.response monkeypatch.setattr('requests.sessions.Session', session_mock) monkeypatch.setattr('requests.Session', session_mock) return session_mock @pytest.fixture(autouse=True) def mock_snowflake_db(monkeypatch, mock_config): """Mock for Snowflake database client.""" snowflake_db_mock = MagicMock() snowflake_sql_executor_class = snowflake_db_mock.SnowflakeSQLExecutor snowflake_sql_executor = snowflake_db_mock.snowflake_sql_executor snowflake_sql_executor_class.return_value.__enter__.return_value = ( snowflake_sql_executor) sql_loader_class = snowflake_db_mock.SQLLoader sql_loader = snowflake_db_mock.sql_loader sql_loader_class.return_value = sql_loader monkeypatch.setattr( snowflake_db, 'SnowflakeSQLExecutor', snowflake_sql_executor_class) monkeypatch.setattr( snowflake_db, 'SQLLoader', sql_loader_class) mock_config['SNOWFLAKE_CONFIG'] = { 'account': 'account', 'db': 'db', 'password': 'password', 'role': 'role', 'schema': 'schema', 'user': 'user', 'warehouse': 'warehouse' } return snowflake_db_mock @pytest.fixture(autouse=True) def mock_config(monkeypatch): """Automatically mock config.""" class ConfigMock: def __init__(self): for key in dir(base_config): if key.startswith('_'): continue self[key] = MagicMock(name=key) self['environment'] = 'dev' def __setitem__(self, key, value): monkeypatch.setattr(base_config, key, value) return ConfigMock() @pytest.fixture(autouse=True) def mock_aws_config(mock_config): """Automatically mock aws config.""" aws_config_mock = MagicMock(name='aws_config') mock_config['aws'] = { 'access_key': aws_config_mock.access_key, 'access_secret': aws_config_mock.access_secret} return aws_config_mock @pytest.fixture(autouse=True) def mock_redshift_config(mock_config): """Automatically mock redshift config.""" redshift_config_mock = MagicMock(name='redshift_config') mock_config['redshift'] = { 'driver': redshift_config_mock.driver, 'user': redshift_config_mock.user, 'password': redshift_config_mock.password, 'host': redshift_config_mock.host, 'port': redshift_config_mock.port, 'db': redshift_config_mock.db} return redshift_config_mock @pytest.fixture(autouse=True) def sf_config_mock(): """Fixture returning the dict with Snowflake connection params.""" return { 'account': 'test_acc', 'role': 'test_role', 'host': 'test_host', 'warehouse': 'test_wh', 'port': 10, 'user': 'test_user', 'password': 'test_pass', 'db': 'test_db', 'schema': 'test_schema' } @pytest.fixture(autouse=True) def mock_s3_tasks(monkeypatch): """Mock s3 tasks from garcon.contrib.""" s3_task_mock = MagicMock(name='s3_task') for task_name in [ 'remove_files_from_path', 'create_object', 'join_s3_objects', 'stream_join_s3_objects']: try: monkeypatch.setattr( garcon.contrib.aws.s3, task_name, s3_task_mock) except AttributeError: # if contrib.aws wasn't imported pass return s3_task_mock @pytest.fixture def mock_garcon(monkeypatch): """Mock garcon flow. This mock runs tasks in flow order preserving the context Usage example: >>> flow = apple_flow.Flow() >>> mock_garcon.run_flow( ... flow, reload=True, context_date_range='2016-09-28_2016-10-05', ... stop_after='check_sos_reports_ingested') It allows to pass any context to the flow as kwargs of `run_flow`. When flow is stopped, the final context is stored in `context` attribute: >>> assert mock_garcon.context == { ... 'reload': True, ... 'bootstrap.reload': True, ... 'bootstrap.date_range': { ... 'start_date': '2016-09-28', ... 'end_date': '2016-10-05'}} All activities are mocked, so you can test if specific loggers were called >>> mock_garcon.activities['bootstrap'].logger.info.assert_called_with( ... 'Active numbers were extracted with no errors') Tasks in a flow are running synchronously. If `stop_after` is passed, flow stops when activity with specified name is done. """ garcon_mock = MagicMock(name='garcon_mock') monkeypatch.setattr('garcon.activity.create', garcon_mock.activity_creator) garcon_mock.activity_creator.return_value = garcon_mock.get_activity garcon_mock.context = {} garcon_mock.activities = {} class StopFlow(Exception): pass def run_activity(activity): for task in garcon.task.flatten( activity.runner.tasks, garcon_mock.context): activity.heartbeat() task_context = dict( list(activity.result.items()) + list(garcon_mock.context.items())) resp = task(task_context, activity=activity) activity.result.update(resp or dict()) def get_activity(name, tasks, generators=None, **kwargs): activity = MagicMock( name=name, runner=tasks, result={}, generators=generators, kwargs=kwargs) garcon_mock.activities[name] = activity return activity def schedule(name, activity, *args, **kwargs): run_activity(activity) garcon_mock.context.update(activity.result) if name == garcon_mock.stop_after: raise StopFlow return activity def run_flow(flow, stop_after=None, **context): garcon_mock.context = context garcon_mock.stop_after = stop_after try: flow.decider(schedule, garcon_mock.context) except StopFlow: pass garcon_mock.schedule.side_effect = schedule garcon_mock.run_flow.side_effect = run_flow garcon_mock.get_activity.side_effect = get_activity return garcon_mock @pytest.fixture(scope='function') def sos_labelid_filter_mock(monkeypatch): """Return mock of util.db.Query class.""" function_result_mock = Mock() function_result_mock.clause = Mock() function_result_mock.values = Mock() function_mock = Mock(return_value=( function_result_mock.clause, function_result_mock.value)) monkeypatch.setattr( 'analytics_aggregation.util.common.sos_labelid_filter', function_mock) return function_result_mock @pytest.fixture(autouse=True) def snowflake_connector_mock(monkeypatch): """Patch Snowflake connector to prevent real calls.""" connect_mock = MagicMock() monkeypatch.setattr(connector, 'connect', connect_mock) return connect_mock