"""Test dynamo_upload module.""" import json from unittest import mock from unittest.mock import MagicMock import pytest from dim_refresh_etl.flows.dynamo_sync.consts import TableEntityTypes from dim_refresh_etl.flows.dynamo_sync.dynamo_upload import dynamo_upload from dim_refresh_etl.flows.dynamo_sync.dynamo_upload.dynamo_upload import boto3 from dim_refresh_etl.flows.dynamo_sync.dynamo_upload.dynamo_upload import \ upload_utils @pytest.fixture def dynamo_table_batch_writer_mock(mocker): """Mock DynamoDB table.""" dynamodb_resource = mocker.patch.object(boto3, 'resource').return_value dynamodb_table = dynamodb_resource.Table.return_value return dynamodb_table.batch_writer.return_value.__enter__() @pytest.fixture def put_in_dynamo_mock(mocker): """Mock _put_in_dynamo function.""" return mocker.patch.object(dynamo_upload, '_put_in_dynamo') @pytest.fixture def time_mock(mocker): """Mock time module.""" return mocker.patch.object(dynamo_upload, 'time') @pytest.fixture def mock_activity(): """Return mock activity.""" activity = MagicMock() activity.return_value.logger = MagicMock() activity.return_value.logger.return_value.info = MagicMock() activity.return_value.logger.return_value.error = MagicMock() return activity class LoggerMock: """.logger mock.""" info = MagicMock() error = MagicMock() class ActivityMock: """Activity mock.""" logger = LoggerMock class TestError(Exception): """Custom test error.""" pass @pytest.fixture def analytics_metadata_unload_settings_mock(mocker): """Mock upload settings.""" config_mock = mocker.patch.object(dynamo_upload, 'upload_config') config_mock.model_type_settings = { TableEntityTypes.analytics_metadata: {'processes_number': 1, 'threads_number': 2}} @pytest.fixture def s3_keys_for_upload(mocker): """Mock _get_s3_keys_for_upload function.""" get_s3_keys_mock = mocker.patch.object( dynamo_upload, '_get_s3_keys_for_upload') get_s3_keys_mock.return_value = [['key1'], ['key2']] def test_start_upload( analytics_metadata_unload_settings_mock, s3_keys_for_upload, put_in_dynamo_mock, time_mock): """Test upload for handling errors during write operations.""" model_type = TableEntityTypes.analytics_metadata s3_unload_link = 's3_unload_link' put_in_dynamo_mock.side_effect = TestError() with pytest.raises(TestError): dynamo_upload.start_upload(model_type, s3_unload_link) def test__run_upload( analytics_metadata_unload_settings_mock, put_in_dynamo_mock): """Test _run_upload function.""" s3_keys = ['key1', 'key2'] model_type = TableEntityTypes.analytics_metadata dynamo_upload._run_upload(s3_keys, model_type) put_in_dynamo_mock.assert_has_calls( [mock.call(model_type, 'key1'), mock.call(model_type, 'key2')]) def test__try_put_in_dynamo_only_fails(put_in_dynamo_mock, time_mock): """Test _try_put_in_dynamo when write operations always fail.""" model_type, s3_path = 'model_type', 's3_path' attempts_left = 5 test_error = TestError() put_in_dynamo_mock.side_effect = test_error result = dynamo_upload._try_put_in_dynamo( model_type, s3_path) assert result == test_error put_in_dynamo_mock.assert_has_calls( [mock.call(model_type, s3_path)] * attempts_left) def test__try_put_in_dynamo_fails_once(put_in_dynamo_mock, time_mock): """Test _try_put_in_dynamo when write operations fail once.""" model_type, s3_path = 'model_type', 's3_path' test_error = TestError() put_in_dynamo_mock.side_effect = [test_error, 'success execution'] result = dynamo_upload._try_put_in_dynamo( model_type, s3_path) assert result is None put_in_dynamo_mock.assert_has_calls([mock.call( model_type, s3_path)] * 2) def test__put_in_dynamo(mocker, dynamo_table_batch_writer_mock): """Test _put_in_dynamo function for proper write calls.""" # Mocking key_s3_path = 'test_s3_path' rows_number = 5 get_rows = mocker.patch.object(dynamo_upload, '_get_rows') row_mock = '{"test": "json"}' get_rows.return_value = [row_mock] * rows_number delete_s3_key = mocker.patch.object(upload_utils, 'delete_s3_key') # Tested function call dynamo_upload._put_in_dynamo('', key_s3_path) # Checks dynamo_table_batch_writer_mock.put_item.assert_has_calls( [mock.call(Item=json.loads(row_mock))] * rows_number) delete_s3_key.assert_called_once_with(key_s3_path)