"""Lambda test module.""" import base64 import itertools import json from unittest.mock import patch from aws_kinesis_agg import aggregator from aws_kinesis_agg import deaggregator import pytest from src import app from src.constants import errors from src.constants import maxwell from tests.consts import common def generate_record(agg_record): """Generate fake kinesis record.""" if agg_record: pk, ehk, data = agg_record.get_contents() result = { 'kinesis': { 'kinesisSchemaVersion': '1.0', 'approximateArrivalTimestamp': 1545084650.987, 'data': base64.b64encode(data).decode('utf-8'), 'partitionKey': pk, 'sequenceNumber': ehk, }} return result @pytest.fixture def kinesis_records(maxwell_records): """Aggregate data and pack into fake kinesis records.""" kinesis_agg = aggregator.RecordAggregator() agg_records = [] for key, data in maxwell_records.items(): json_data = json.dumps(data).encode('utf-8') result = kinesis_agg.add_user_record(key, json_data) if result: fake_kinesis_record = generate_record(result) agg_records.append(fake_kinesis_record) fake_kinesis_record = generate_record(kinesis_agg.clear_and_get()) agg_records.append(fake_kinesis_record) return agg_records @pytest.fixture def field_name(): """Get fake maxwell data field name.""" return common.DATA_FIELD @pytest.fixture def maxwell_tables(): """Get list of tables handled by maxwell.""" return [ common.ENCODING_QUEUE_DETAIL, common.ENCODING_QUEUE ] @pytest.fixture def maxwell_records(maxwell_tables, field_name): """Get list of fake maxwell records.""" maxwell_records = {} type_iter = itertools.cycle( [maxwell.TYPE_INSERT, maxwell.TYPE_UPDATE, maxwell.TYPE_DELETE]) for i in range(0, len(maxwell_tables) * 3): maxwell_records[str(i)] = { maxwell.MAXWELL_TABLE: maxwell_tables[i // 3], maxwell.MAXWELL_TYPE: next(type_iter), maxwell.MAXWELL_DATA: {field_name: i} } maxwell_records[str(len(maxwell_records))] = { maxwell.MAXWELL_TABLE: maxwell_tables[0], maxwell.MAXWELL_DATA: {field_name: 'test'} } return maxwell_records @pytest.fixture def event(kinesis_records): """Event Records.""" return { maxwell.EVENT_RECORDS: kinesis_records } @pytest.fixture def empty_event(): """Empty Records Event.""" return {maxwell.EVENT_RECORDS: []} @patch('src.app.logger') def test_deaggregate_records(mock_logger, event, monkeypatch): """Test deaggregate_records function.""" expected_result = { common.ENCODING_QUEUE_DETAIL: [ { maxwell.MAXWELL_TABLE: common.ENCODING_QUEUE_DETAIL, maxwell.MAXWELL_TYPE: maxwell.TYPE_INSERT, maxwell.MAXWELL_DATA: { common.DATA_FIELD: 0 } }, { maxwell.MAXWELL_TABLE: common.ENCODING_QUEUE_DETAIL, maxwell.MAXWELL_TYPE: maxwell.TYPE_UPDATE, maxwell.MAXWELL_DATA: { common.DATA_FIELD: 1 } }, { maxwell.MAXWELL_TABLE: common.ENCODING_QUEUE_DETAIL, maxwell.MAXWELL_TYPE: maxwell.TYPE_DELETE, maxwell.MAXWELL_DATA: { common.DATA_FIELD: 2 } } ] } result = app.deaggregate_records(event[maxwell.EVENT_RECORDS]) mock_logger.debug.assert_any_call("Deaggregated 3 Maxwell's Daemon records.") mock_logger.warning.assert_any_call(errors.SKIPPING_RECORD, common.ENCODING_QUEUE, maxwell.TYPE_INSERT) mock_logger.warning.assert_any_call(errors.SKIPPING_RECORD, common.ENCODING_QUEUE, maxwell.TYPE_UPDATE) mock_logger.warning.assert_any_call(errors.SKIPPING_RECORD, common.ENCODING_QUEUE, maxwell.TYPE_DELETE) mock_logger.error.assert_any_call(errors.MISSING_ITEM, maxwell.MAXWELL_TYPE) for record in deaggregator.iter_deaggregate_records(event[maxwell.EVENT_RECORDS]): record_data = json.loads( base64.b64decode(record[maxwell.RECORD_KINESIS][maxwell.KINESIS_DATA]).decode('utf-8') ) mock_logger.debug.assert_any_call( f'json data from kinesis {record_data}' ) assert result == expected_result @patch('src.app.logger') def test_deaggregate_records_empty_list(mock_logger): """Test deaggregate_records function in case of empty list.""" expected_result = {} result = app.deaggregate_records([]) mock_logger.debug.assert_any_call("Deaggregated 0 Maxwell's Daemon records.") assert result == expected_result @patch('src.app.logger') @patch('src.app.deaggregate_records') @patch('src.app.log_records') def test_handler(mock_log_records, mock_deaggregate_records, mock_logger, event): """Test handler function.""" mock_return_value = ['record1', 'record2', 'record3'] mock_deaggregate_records.return_value = mock_return_value result = app.handler(event, None) mock_logger.debug.assert_called_once_with(event) mock_deaggregate_records.assert_called_once_with(event[maxwell.EVENT_RECORDS]) mock_log_records.assert_called_once_with(mock_return_value) assert result == {'status': 'OK'} @patch('src.app.logger') @patch('src.app.deaggregate_records') @patch('src.app.log_records') def test_handler_empty_records(mock_log_records, mock_deaggregate_records, mock_logger, empty_event): """Test handler function with empty records in event.""" mock_return_value = [] mock_deaggregate_records.return_value = mock_return_value result = app.handler(empty_event, None) mock_logger.debug.assert_called_once_with(empty_event) mock_deaggregate_records.assert_called_once_with(empty_event[maxwell.EVENT_RECORDS]) mock_log_records.assert_called_once_with(mock_return_value) assert result == {'status': 'OK'} @patch('src.app.logger') @patch('src.app.deaggregate_records') @patch('src.app.log_records') def test_handler_empty_event(mock_log_records, mock_deaggregate_records, mock_logger): """Test handler function with empty event.""" result = app.handler({}, None) mock_logger.error.assert_called_once_with(errors.MISSING_ITEM, maxwell.EVENT_RECORDS) mock_deaggregate_records.assert_not_called() mock_log_records.assert_not_called() assert result is None @patch('src.app.logger') @patch('src.app.capture_exception') @patch('src.app.deaggregate_records') @patch('src.app.log_records') def test_handler_exception(mock_log_records, mock_deaggregate_records, mock_capture_exception, mock_logger, event): """Test handler function when an exception is raised.""" mock_deaggregate_records.side_effect = ValueError('Test Exception') with pytest.raises(ValueError, match='Test Exception'): app.handler(event, None) mock_capture_exception.assert_called_once_with(mock_deaggregate_records.side_effect) mock_logger.exception.assert_called_once_with('Test Exception') mock_deaggregate_records.assert_called_once_with(event[maxwell.EVENT_RECORDS]) mock_log_records.assert_not_called()