"""Test Queues work.""" import json from queue import Queue from time import sleep from unittest.mock import patch from unittest.mock import MagicMock from uuid import uuid4 import pytest from sales.constants.exceptions import FailedToStartError from tests.conftest import EXECUTORS_COUNT, PREFETCH_NUMBER, READERS_COUNT from tests.conftest import STOP_TIMEOUT, VISIBILITY_TIMEOUT PREFETCH_QUEUE_GET_TIMEOUT = 2 def add_messages_to_queue(sqs_queue, count): """Add {count} messages to the queue.""" while count > 0: body = json.dumps({'foo': str(uuid4())}) sqs_queue.send_message(MessageBody=body) count -= 1 def test_read_message(queue_reader_worker, prefetch_queue, sqs_queue): """Verify we can read all messages, sent to queue.""" add_messages_to_queue(sqs_queue, PREFETCH_NUMBER) queue_reader_worker.start() sleep(1) queue_reader_worker.stop(STOP_TIMEOUT) counter = 0 while prefetch_queue.qsize(): item = prefetch_queue.get(False, timeout=PREFETCH_QUEUE_GET_TIMEOUT) if item is None: break counter += 1 assert counter == PREFETCH_NUMBER def test_read_message_prefetch_queue_full( queue_reader_worker, prefetch_queue, sqs_queue): """Test all messages readability after full prefetch. Verify we can read all messages, sent to queue, assuming even if the prefetch queue becomes full """ add_messages_to_queue(sqs_queue, PREFETCH_NUMBER * 2) queue_reader_worker.start() sleep(1) counter = 0 while counter < PREFETCH_NUMBER * 2: prefetch_queue.get(timeout=PREFETCH_QUEUE_GET_TIMEOUT) counter += 1 assert counter == PREFETCH_NUMBER * 2 @patch('moto.sqs.models.Message.change_visibility') def test_read_message_prefetch_queue_full_error( change_visibility_mock, queue_reader_worker, sqs_queue): """Verify message deletion from SQS after prefetch. Verify we return the message back to the SQS queue, if prefetch queue becomes full during the prefetch process """ add_messages_to_queue(sqs_queue, 2) queue_reader_worker.prefetch_queue = Queue(1) queue_reader_worker.num_messages = 2 queue_reader_worker.start() sleep(1) queue_reader_worker.stop(STOP_TIMEOUT) assert change_visibility_mock.called change_visibility_mock.assert_called_with(VISIBILITY_TIMEOUT) def test_start(worker_manager, sqs_queue): """Verify that the manager can start a defined number of workers.""" worker_manager.start() assert worker_manager.readers_started == READERS_COUNT assert worker_manager.executors_started == EXECUTORS_COUNT worker_manager.stop(STOP_TIMEOUT) @patch('boto3.resource') def test_start_failed(mock_boto, worker_manager): """Verify FailedToStartError is raised properly. Verify that FailedToStartError is raised if no workers were able to start successfully. """ mock_boto.return_value = None with pytest.raises(FailedToStartError): worker_manager.start() @patch('sales.logic.queue.process_message', side_effect=Exception) @patch('sales.models.export_request.ExportRequest.add_error') def test_exception_in_handler( mock_add_error, mock_process_message, prefetch_queue, app_execution_worker, valid_message_body): """Test handle_message when process_message raises.""" message = MagicMock() message.body = json.dumps(valid_message_body) message.delete = MagicMock() prefetch_queue.put(message) app_execution_worker.prefetch_queue = prefetch_queue app_execution_worker.start() sleep(1) app_execution_worker.stop(STOP_TIMEOUT) assert mock_add_error.call_count == 1 @patch('sales.logic.queue.process_message') @patch('sales.models.export_request.ExportRequest.validate') def test_process_message_success( mock_validate, mock_process_message, prefetch_queue, app_execution_worker, valid_message_body): """Verify that we delete the message from SQS after processing it.""" message = MagicMock(return_value=None) message.body = json.dumps(valid_message_body) message.delete = MagicMock() prefetch_queue.put(message) app_execution_worker.prefetch_queue = prefetch_queue app_execution_worker.start() sleep(1) app_execution_worker.stop(STOP_TIMEOUT) assert message.delete.call_count == 1 @patch('sales.logic.queue.process_message') def test_invalid_message( mock_process_message, prefetch_queue, app_execution_worker, valid_message_body): """Test an invalid message.""" valid_message_body['foo'] = 'bar' message = MagicMock(return_value=None) message.body = json.dumps(valid_message_body) message.delete = MagicMock() prefetch_queue.put(message) app_execution_worker.prefetch_queue = prefetch_queue app_execution_worker.start() sleep(1) assert mock_process_message.call_count == 0 @patch('sales.logic.queue.request') def test_ows_call( mock_request, prefetch_queue, app_execution_worker, valid_message_body): """Test the call sent to owsrequest.""" mock_process = MagicMock() mock_request.process = mock_process message = MagicMock(return_value=None) message.body = json.dumps(valid_message_body) message.delete = MagicMock() prefetch_queue.put(message) app_execution_worker.prefetch_queue = prefetch_queue app_execution_worker.start() sleep(1) app_execution_worker.stop(STOP_TIMEOUT) assert mock_process.call_count == 1 assert mock_process.call_args[0][0] == 'daemon-sales-ingest' assert mock_process.call_args[0][1] == 'test' assert mock_process.call_args[0][2] == 'POST' assert mock_process.call_args[0][3] == 'ows-royalties-workflows' assert mock_process.call_args[0][4] == '/endpoint'