"""Test Queues work.""" from queue import Queue from time import sleep from boto.sqs import connect_to_region from flexmock import flexmock from moto import mock_sqs import pytest from label_copy_export.connectors import sqs from label_copy_export.connectors.loggly import app_logger from label_copy_export.connectors.sqs import JSONMessageExt from label_copy_export.constants.exceptions import FailedToStartError from label_copy_export.logic import queue from tests.utils import test_sqs_utils PREFETCH_NUMBER = 2 VISIBILITY_TIMEOUT = 60 WAIT_TIME_SECONDS = 2 NUM_MESSAGES = 4 RETURN_TO_QUEUE_TIMEOUT = 1 MAX_RETRIES = 10 STOP_TIMEOUT = 5 PREFETCH_QUEUE_GET_TIMEOUT = 5 SQS_QUEUE_NAME = 'test-q' KEY_FILE = 'key' READERS_COUNT = 2 EXECUTORS_COUNT = 2 @pytest.fixture(scope='session', autouse=True) def start_sqs_mock(request): """Use this function for SQS mock.""" mock = mock_sqs() mock.start() def start_sqs_mock_teardown(): mock.stop() request.addfinalizer(start_sqs_mock_teardown) @pytest.fixture def prefetch_queue(): """Prefetch queue with this fucntion.""" pref_queue = Queue(PREFETCH_NUMBER) return pref_queue @pytest.fixture() def sqs_queue(request): """Connect to SQS queue.""" conn = sqs.get_connection( access_key_id='test', secret_access_key='test_secret') conn.create_queue(SQS_QUEUE_NAME) mock_queue = sqs.get_queue(SQS_QUEUE_NAME, sqs_connection=conn) def sqs_queue_teardown(): mock_queue.delete() request.addfinalizer(sqs_queue_teardown) return mock_queue @pytest.fixture def queue_reader_worker(request, prefetch_queue, sqs_queue): """Create queue reader worker.""" worker = queue.QueueReaderWorker( 'test-worker', prefetch_queue, sqs_queue, VISIBILITY_TIMEOUT, WAIT_TIME_SECONDS, NUM_MESSAGES, RETURN_TO_QUEUE_TIMEOUT, MAX_RETRIES, app_logger ) worker.debug = True def queue_reader_worker_teardown(): worker.stop(STOP_TIMEOUT) request.addfinalizer(queue_reader_worker_teardown) return worker @pytest.fixture def api_execution_worker(request, prefetch_queue, sqs_queue): """Create queue executive worker.""" worker = queue.CsvGenerationWorker( 'test-worker', prefetch_queue, sqs_queue, RETURN_TO_QUEUE_TIMEOUT, app_logger ) worker.debug = True def api_execution_worker_teardown(): worker.stop(STOP_TIMEOUT) request.addfinalizer(api_execution_worker_teardown) return worker @pytest.fixture def worker_manager(): """Create queue worker manager.""" manager = queue.QueueWorkerManager( READERS_COUNT, EXECUTORS_COUNT, SQS_QUEUE_NAME, PREFETCH_NUMBER, VISIBILITY_TIMEOUT, WAIT_TIME_SECONDS, NUM_MESSAGES, app_logger ) manager.debug = True return manager @pytest.fixture def test_connection(): """Make mock connection.""" test_conn = connect_to_region( 'us-east-1', aws_access_key_id='test', aws_secret_access_key='test_secret') flexmock(sqs).should_receive('get_connection').and_return(test_conn) return test_conn def test_read_message(queue_reader_worker, prefetch_queue, sqs_queue): """Verify we can read all messages, sent to queue.""" test_sqs_utils.put_sqs_messages(sqs_queue, PREFETCH_NUMBER) queue_reader_worker.start() counter = 0 while counter < PREFETCH_NUMBER: prefetch_queue.get(timeout=PREFETCH_QUEUE_GET_TIMEOUT) counter += 1 assert counter == PREFETCH_NUMBER def test_read_message_prefetch_queue_full( queue_reader_worker, prefetch_queue, sqs_queue): """Test all messages readability after full prefetch. Verify we can read all messages, sent to queue, assuming even if the prefetch queue becomes full """ test_sqs_utils.put_sqs_messages(sqs_queue, PREFETCH_NUMBER * 2) queue_reader_worker.start() counter = 0 while counter < PREFETCH_NUMBER * 2: prefetch_queue.get(timeout=PREFETCH_QUEUE_GET_TIMEOUT) counter += 1 assert counter == PREFETCH_NUMBER * 2 def test_read_message_prefetch_queue_full_error( queue_reader_worker, sqs_queue): """Verify message deletion from SQS after prefetch. Verify we return the message back to the SQS queue, if prefetch queue becomes full during the prefetch process """ (flexmock(JSONMessageExt).should_receive('change_visibility'). at_least().once()) test_sqs_utils.put_sqs_messages(sqs_queue, 2) queue_reader_worker.prefetch_queue = Queue(1) queue_reader_worker.num_messages = 2 queue_reader_worker.start() sleep(3) def test_start(worker_manager, test_connection): """Verify that the manager can start a defined number of workers.""" test_connection.create_queue(SQS_QUEUE_NAME) worker_manager.start() assert worker_manager.readers_started == READERS_COUNT assert worker_manager.executors_started == EXECUTORS_COUNT worker_manager.stop(STOP_TIMEOUT) test_connection.get_queue(SQS_QUEUE_NAME).delete() def test_start_failed(worker_manager, test_connection): """Verify FailedToStartError is raised properly. Verify that FailedToStartError is raised if no workers were able to start successfully. """ with pytest.raises(FailedToStartError): worker_manager.start()