"""Unit tests for Garcon CLI for backload_executor.py.""" import json from unittest.mock import MagicMock, patch import pytest from feed_ingestion.bin import backload_executor @pytest.fixture() def mock_backload_executor_tasks(): """Mock tasks in task_queue targeted for test_flow flow.""" total_tasks = 10 date = '2020-01-01' with patch('feed_ingestion.bin.backload_executor' '.BackloadTasksExecutor') as mock_backload_executor: mock_select_flows_to_process = mock_backload_executor.return_value.\ __enter__.return_value.select_flows_to_process mock_select_flows_to_process.return_value = \ [('test_flow', total_tasks)] def select_backload_tasks_by_flow(flow, limit): return [ ('theorchard', date, '{}') for _ in range(min(limit, total_tasks)) ] mock_select_backload_tasks_by_flow = \ mock_backload_executor.return_value.__enter__\ .return_value.select_backload_tasks_by_flow \ = MagicMock(new=select_backload_tasks_by_flow) mock_select_backload_tasks_by_flow.side_effect = \ select_backload_tasks_by_flow yield mock_backload_executor @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @pytest.mark.parametrize('limit, num_running_tasks, calls_expected', [ (8, 10, 0), (8, 8, 0), (8, 2, 6), (8, 0, 8), ]) def test_start_tasks( mock_count_running_workflows_by_type, mock_get_flow, mock_backload_executor_tasks, limit, num_running_tasks, calls_expected): """Test limits.""" # mock LIMITS mock_count_running_workflows_by_type.return_value = num_running_tasks mock_get_flow.domain = 'test' mock_get_flow.name = 'test_flow' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call, \ patch('feed_ingestion.bin.backload_executor._flow_limit') \ as mock_limit: mock_limit.return_value = limit mock_call.return_value = 0 backload_executor.start_tasks() assert mock_call.call_count == calls_expected @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @pytest.mark.parametrize('only_flows, calls_expected', [ (['test_flow'], 7), (['other_flow'], 0), (['other_flow', 'also_other_flow'], 0), (['other_flow', 'test_flow'], 7), ]) def test_start_tasks_limit_only( mock_count_running_workflows_by_type, mock_get_flow, mock_backload_executor_tasks, only_flows, calls_expected): """Test when only limited flows are processing.""" mock_count_running_workflows_by_type.return_value = 3 mock_get_flow.domain = 'test' mock_get_flow.name = 'test_flow' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call, \ patch('feed_ingestion.bin.backload_executor._flow_limit') \ as mock_limit: mock_limit.return_value = 10 mock_call.return_value = 0 backload_executor.start_tasks(only_flows=only_flows) assert mock_call.call_count == calls_expected def test_flow_limit(): """Test _flow_limit.""" backload_executor.LIMIT_CONCURRENT_EXECUTIONS_BY_FLOW = { 'specific_flow': 10, 'DEFAULT': 5, } assert backload_executor._flow_limit('specific_flow') == 10 assert backload_executor._flow_limit('other_flow') == 5 @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') def test_start_tasks_adds_backfill_to_context( mock_count_running_workflows_by_type, mock_get_flow, mock_backload_executor_tasks): """Test that backfill parameter is added to the context.""" mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'test_flow' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call, \ patch('feed_ingestion.bin.backload_executor._flow_limit') \ as mock_limit: mock_limit.return_value = 1 mock_call.return_value = 0 backload_executor.start_tasks() call_args = mock_call.call_args[0][0] context_index = call_args.index('--context') + 1 context = json.loads(call_args[context_index]) assert context['backfill'] == 'True'