"""Test throttlers.""" import random import time from unittest.mock import call import sentry_sdk from freezegun import freeze_time import psutil import pytest from contextlib import nullcontext as does_not_raise from freezegun.api import FakeDatetime from video import config from video.connectors import cloudwatch as cloudwatch_connector from video.constants.exceptions import NotCurrentlyAcceptingWork from video.logic import activity_task_worker_throttlers @pytest.mark.parametrize( ( 'test_description', 'put_metric_data_exception', 'expected_sentry_capture_exception_calls', ), [ ( 'no put_metric_data exception', None, [], ), ( 'with put_metric_data exception', Exception('Test exception'), [call()], ), ], ) @freeze_time('2019-05-11 09:50:00') def test__log_custom_metrics_to_cloudwatch( mocker, test_description, put_metric_data_exception, expected_sentry_capture_exception_calls, ): """Test _log_custom_metrics_to_cloudwatch.""" mocker.patch.object( cloudwatch_connector, 'get_cloudwatch_client', autospec=True, ).return_value.put_metric_data.side_effect = put_metric_data_exception mocker.patch.object(sentry_sdk, 'capture_exception', autospec=True) test_cpu_utilization_percent = 42.0 test_memory_utilization_percent = 84.0 activity_task_worker_throttlers._log_custom_metrics_to_cloudwatch( test_cpu_utilization_percent, test_memory_utilization_percent, ) assert cloudwatch_connector.get_cloudwatch_client.return_value.put_metric_data.mock_calls == [ call( Namespace=config.CLOUDWATCH_NAMESPACE, MetricData=[ { 'MetricName': config.CUSTOM_METRIC_CPU_USAGE_PERCENT, 'Dimensions': [ { 'Name': 'Environment', 'Value': config.ENVIRONMENT }, ], 'Timestamp': FakeDatetime(2019, 5, 11, 9, 50), 'Value': test_cpu_utilization_percent, 'Unit': 'Percent', 'StorageResolution': 1, }, { 'MetricName': config.CUSTOM_METRIC_MEMORY_USAGE_PERCENT, 'Dimensions': [ { 'Name': 'Environment', 'Value': config.ENVIRONMENT }, ], 'Timestamp': FakeDatetime(2019, 5, 11, 9, 50), 'Value': test_memory_utilization_percent, 'Unit': 'Percent', 'StorageResolution': 1, }, ] ) ] assert sentry_sdk.capture_exception.mock_calls == expected_sentry_capture_exception_calls @pytest.mark.parametrize( ( 'test_description', 'cpu_percent', 'memory_percent', 'expected_result', ), [ ( 'under resource limit', config.MAX_ALLOWED_CPU_UTILIZATION_PERCENT, config.MAX_ALLOWED_MEMORY_UTILIZATION_PERCENT, True, ), ( 'over cpu limit', config.MAX_ALLOWED_CPU_UTILIZATION_PERCENT + 1, config.MAX_ALLOWED_MEMORY_UTILIZATION_PERCENT, False, ), ( 'over memory limit', config.MAX_ALLOWED_CPU_UTILIZATION_PERCENT, config.MAX_ALLOWED_MEMORY_UTILIZATION_PERCENT + 1, False, ), ( 'over cpu and memory limits', config.MAX_ALLOWED_CPU_UTILIZATION_PERCENT + 1, config.MAX_ALLOWED_MEMORY_UTILIZATION_PERCENT + 1, False, ), ] ) def test__is_resource_utilization_ok( mocker, test_description, cpu_percent, memory_percent, expected_result, ): """Test _is_resource_utilization_ok.""" mocker.patch.object(activity_task_worker_throttlers, '_log_custom_metrics_to_cloudwatch', autospec=True) mocker.patch.object(psutil, 'cpu_percent', autospec=True, return_value=cpu_percent) mocker.patch.object(psutil, 'virtual_memory', autospec=True).return_value.percent = memory_percent assert activity_task_worker_throttlers._is_resource_utilization_ok() == expected_result @pytest.mark.parametrize( ( 'test_description', '_is_resource_utilization_ok_return_value', 'expected_raise', ), [ ( 'under resource limit', True, does_not_raise(), ), ( 'over resource limit', False, pytest.raises(NotCurrentlyAcceptingWork), ), ]) def test_handling_worker_throttler( mocker, test_description, _is_resource_utilization_ok_return_value, expected_raise, ): """Test handling_worker_throttler.""" mocker.patch.object( activity_task_worker_throttlers, '_is_resource_utilization_ok', autospec=True, return_value=_is_resource_utilization_ok_return_value, ) mocker.patch.object(time, 'sleep', autospec=True) mocker.patch.object(random, 'random', autospec=True, return_value=0.5) with expected_raise: assert activity_task_worker_throttlers.handling_worker_throttler() is None assert time.sleep.mock_calls == [call(config.MAX_TIMEOUT_BETWEEN_PICKING_UP_WORK_AND_HANDLING_SECONDS * 0.5)] @pytest.mark.parametrize( ( 'test_description', '_is_resource_utilization_ok_return_values', 'expected_sleep_calls', ), [ ( 'under resource limit', [ True, ], [], ), ( 'over resource limit, under resource limit', [ False, True, ], [ call(config.MAX_TIMEOUT_BETWEEN_RESOURCE_UTILIZATION_CHECKS_SECONDS * 0.5), ], ), ( 'over resource limit, over resource limit, under resource limit', [ False, False, True, ], [ call(config.MAX_TIMEOUT_BETWEEN_RESOURCE_UTILIZATION_CHECKS_SECONDS * 0.5), call(config.MAX_TIMEOUT_BETWEEN_RESOURCE_UTILIZATION_CHECKS_SECONDS * 0.5), ], ), ]) def test_is_polling_worker_activated( mocker, test_description, _is_resource_utilization_ok_return_values, expected_sleep_calls, ): """Test is_polling_worker_activated.""" mocker.patch.object( activity_task_worker_throttlers, '_is_resource_utilization_ok', autospec=True, side_effect=_is_resource_utilization_ok_return_values, ) mocker.patch.object(time, 'sleep', autospec=True) mocker.patch.object(random, 'random', autospec=True, return_value=0.5) assert activity_task_worker_throttlers.is_polling_worker_activated() is True assert time.sleep.mock_calls == expected_sleep_calls @pytest.mark.parametrize( ( 'test_description', '_is_resource_utilization_ok_return_values', 'expected_sleep_calls', ), [ ( 'under resource limit', [ True, ], [ call(config.MAX_TIMEOUT_BEFORE_PICKING_UP_WORK_SECONDS * 0.5), ], ), ( 'over resource limit, under resource limit', [ False, True, ], [ call(config.MAX_TIMEOUT_BEFORE_PICKING_UP_WORK_SECONDS * 0.5), call(config.MAX_TIMEOUT_BETWEEN_RESOURCE_UTILIZATION_CHECKS_SECONDS * 0.5), ], ), ( 'over resource limit, over resource limit, under resource limit', [ False, False, True, ], [ call(config.MAX_TIMEOUT_BEFORE_PICKING_UP_WORK_SECONDS * 0.5), call(config.MAX_TIMEOUT_BETWEEN_RESOURCE_UTILIZATION_CHECKS_SECONDS * 0.5), call(config.MAX_TIMEOUT_BETWEEN_RESOURCE_UTILIZATION_CHECKS_SECONDS * 0.5), ], ), ]) def test_is_handling_worker_activated( mocker, test_description, _is_resource_utilization_ok_return_values, expected_sleep_calls, ): """Test is_handling_worker_activated.""" mocker.patch.object( activity_task_worker_throttlers, '_is_resource_utilization_ok', autospec=True, side_effect=_is_resource_utilization_ok_return_values, ) mocker.patch.object(time, 'sleep', autospec=True) mocker.patch.object(random, 'random', autospec=True, return_value=0.5) assert activity_task_worker_throttlers.is_handling_worker_activated() is True assert time.sleep.mock_calls == expected_sleep_calls