import types import pytest from unittest.mock import MagicMock from octopus.flows.dim_refresh import generator from sqlalchemy.engine import Engine def test_get_min_max_list(monkeypatch): """Generator method should output generator """ context = { 'destination_s3_key': 'dim_label/data/dim_label.txt.gz', 'chunk_size': 100000, 'table': 'vendor', 'bucket': 'prod-staging-dim-imports', 'primary_key': 'id', 'query': '''some query''' } engine = MagicMock() connection = MagicMock() engine.connect.return_value = connection connection.execute.return_value = [ {'min_id': 10, 'max_id': 20}, {'min_id': 20, 'max_id': 30}] monkeypatch.setattr( generator, 'get_engine', MagicMock(return_value=engine)) data = generator.get_min_max_list( context.get('primary_key'), context.get('chunk_size'), context.get('table')) assert generator.get_engine.called assert engine.connect.called assert len(data.get('min_max_pair')) == len( connection.execute.return_value) for i, return_value in enumerate(connection.execute.return_value): assert connection.execute.return_value[i].get( 'min_id') == return_value.get('min_id') assert connection.execute.return_value[i].get( 'max_id') == return_value.get('max_id') def test_missing_primary_key(): with pytest.raises(Exception): generator.get_min_max_list(None, None, None) def test_generator(): """Generator method should output generator """ context = { 'destination_s3_key': 'dim_label/data/dim_label.txt.gz', 'chunk_size': 100000, 'table': 'vendor', 'bucket': 'prod-staging-dim-imports', 'primary_key': 'id', 'query': '''some query''', 'bootstrap.min_max_pair': [ {'min_id': 10, 'max_id': 20}, {'min_id': 20, 'max_id': 30}] } gen = generator.get_generator(context) assert isinstance(gen, types.GeneratorType) # Go through the generator data = list(gen) for i, return_value in enumerate(context.get('bootstrap.min_max_pair')): assert data[i].get('generator.min_id') == return_value.get('min_id') assert data[i].get('generator.max_id') == return_value.get('max_id') def test_primary_key_required_exception(): """ Testing exception thrown when no primary key is provided """ context = {} assert not list(generator.get_generator(context)) def test_get_range_sql_output(): """ Testing the output type of get_range_sql """ primary_key = 'id' chunk_size = 1000 table = 'sometable' assert isinstance(generator.get_range_sql(primary_key, chunk_size, table), str) def test_get_engine(): """ Testing if get_engine method return Engine object """ assert isinstance(generator.get_engine(), Engine)