"""Tests for lambda function module.""" import base64 from collections import defaultdict import itertools import json from aws_kinesis_agg import aggregator import pytest from src import index from src.constants import errors from src.constants import maxwell def generate_record(agg_record): """Generate fake kinesis record.""" if agg_record: pk, ehk, data = agg_record.get_contents() result = { maxwell.RECORD_KINESIS: { 'kinesisSchemaVersion': '1.0', 'approximateArrivalTimestamp': 1545084650.987, maxwell.KINESIS_DATA: base64.b64encode(data), 'partitionKey': pk, 'sequenceNumber': ehk, }} return result @pytest.fixture def kinesis_records(maxwell_records): """Aggregate data and pack into fake kinesis records.""" kinesis_agg = aggregator.RecordAggregator() agg_records = [] for key, data in maxwell_records.items(): result = kinesis_agg.add_user_record( key, json.dumps(data).encode()) if result: fake_kinesis_record = generate_record(result) agg_records.append(fake_kinesis_record) fake_kinesis_record = generate_record(kinesis_agg.clear_and_get()) agg_records.append(fake_kinesis_record) return agg_records @pytest.fixture def field_name(): """Get fake maxwell data field name.""" return 'data_field' @pytest.fixture def maxwell_tables(): """Get list of tables handled by maxwell.""" return [ maxwell.TABLE_ENCODING_QUEUE, maxwell.TABLE_DMS_DELIVERY_SPEC, maxwell.TABLE_ENCODING_QUEUE_DETAIL ] @pytest.fixture def maxwell_records(maxwell_tables, field_name): """Get list of fake maxwell records.""" maxwell_records = {'-1': {}} type_iter = itertools.cycle( [maxwell.TYPE_INSERT, maxwell.TYPE_UPDATE]) for i in range(0, len(maxwell_tables) * 2): maxwell_records[str(i)] = { maxwell.MAXWELL_TABLE: maxwell_tables[i // 2], maxwell.MAXWELL_TYPE: next(type_iter), maxwell.MAXWELL_DATA: {field_name: i} } return maxwell_records @pytest.fixture def processing_records(maxwell_records): """Get list of fake maxwell records that should be processed.""" return [ r for r in maxwell_records.values() if (maxwell.MAXWELL_TABLE in r and r[maxwell.MAXWELL_TABLE] in maxwell.PROCESSING_RECORDS and r[maxwell.MAXWELL_TYPE] in maxwell.PROCESSING_RECORDS[ r[maxwell.MAXWELL_TABLE]]) ] @pytest.fixture def deaggregate_result(processing_records): """Get processing records in deaggregate function result style.""" expected_result = defaultdict(lambda: defaultdict(list)) for r in processing_records: table_name = r[maxwell.MAXWELL_TABLE] change_type = r[maxwell.MAXWELL_TYPE] expected_result[table_name][change_type].append( r[maxwell.MAXWELL_DATA]) return expected_result def test_deaggregate_records( kinesis_records, field_name, maxwell_tables, deaggregate_result, mocker): """Test deaggregate_records function.""" mocked_loging = mocker.patch('src.index.logging') mocked_logger = mocked_loging.logger mocked_error = mocked_logger.error mocked_key_fields = mocker.patch('src.index.key_fields') mocked_key_fields.TABLE_KEY = {tn: [field_name] for tn in maxwell_tables} result = index.deaggregate_records(kinesis_records) mocked_error.assert_called_once_with( errors.MISSING_ITEM, maxwell.MAXWELL_TABLE) assert result == deaggregate_result @pytest.mark.parametrize( 'records, f_call', ( ({}, (0, 0, 0)), ({maxwell.TABLE_ENCODING_QUEUE: {'i': ['r1', 'r2']}}, (1, 0, 0)), ({maxwell.TABLE_DMS_DELIVERY_SPEC: {'i': ['r1', 'r2']}}, (0, 1, 0)), ({maxwell.TABLE_ENCODING_QUEUE_DETAIL: {'i': ['r1', 'r2']}}, (0, 0, 1)), ) ) def test_process_records( records, f_call, mocker): """Test process_records function.""" queue_call, delivery_call, detail_call = f_call mocked_orders = mocker.patch('src.index.orders') mocked_proc_queue = mocked_orders.process_encoding_queue mocked_dms_delivery_spec = mocker.patch('src.index.dms_delivery_spec') mocked_proc_delivery = mocked_dms_delivery_spec.process_dms_delivery_spec mocked_jobs = mocker.patch('src.index.jobs') mocked_proc_detail = mocked_jobs.process_details index.process_records(records) assert mocked_proc_queue.call_count == queue_call if queue_call: assert mocked_proc_queue.call_args[0] == ( records[maxwell.TABLE_ENCODING_QUEUE],) assert mocked_proc_delivery.call_count == delivery_call if delivery_call: assert mocked_proc_delivery.call_args[0] == ( records[maxwell.TABLE_DMS_DELIVERY_SPEC],) assert mocked_proc_detail.call_count == detail_call if detail_call: assert mocked_proc_detail.call_args[0] == ( records[maxwell.TABLE_ENCODING_QUEUE_DETAIL],) @pytest.mark.parametrize( 'event, error_call, proc_call', ( ({}, 1, 0), ({maxwell.EVENT_RECORDS: 'rec'}, 0, 1), ) ) def test_handler(event, error_call, proc_call, mocker): """Test handler function.""" deaggregated_records = ['r1', 'r2'] mocked_loging = mocker.patch('src.index.logging') mocked_logger = mocked_loging.logger mocked_error = mocked_logger.error mocked_deaggregate_records = mocker.patch('src.index.deaggregate_records') mocked_deaggregate_records.return_value = deaggregated_records mocked_process_records = mocker.patch('src.index.process_records') index.handler(event, None) assert mocked_error.call_count == error_call if error_call: assert mocked_error.call_args[0] == ( errors.MISSING_ITEM, maxwell.EVENT_RECORDS) assert mocked_deaggregate_records.call_count == proc_call assert mocked_process_records.call_count == proc_call if proc_call: assert mocked_deaggregate_records.call_args[0] == ( event[maxwell.EVENT_RECORDS],) assert mocked_process_records.call_args[0] == ( deaggregated_records,)