"""Unit tests for Garcon CLI.""" from unittest import mock from unittest.mock import call from unittest.mock import patch from freezegun import freeze_time from garcon_contrib.dynamo_feed_status import \ garcon_feed_status from feed_ingestion.bin import exec_flows from feed_ingestion.flows import base from feed_ingestion.util import task_status @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') def test_exec_multiple_days(mock_count_running_workflows_by_type, mock_get_flow): """Test workflow execution for multiple days.""" mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' # create a pool of all potential calls for the test month call_params = [[ 'garcon', 'exec', 'foo', '--context', '{{"context_date": "2015-09-{:02d}"}}'.format(i)] for i in range( 1, 31)] dry_run_params = [[ 'garcon', 'exec', 'foo', '--context', '\'{{"context_date": "2015-09-{:02d}"}}\''.format(i)] for i in range( 1, 31)] test_cases = [ {'skip': 1, 'days': 5}, # expected range {'skip': 3, 'days': 7}, # unusual skip {'skip': 0, 'days': 3}, # test with no skips {'skip': 0, 'days': 1}, # today {'skip': 1, 'days': 1}, # yesterday {'skip': 10, 'days': 1}, # very specific date in the past ] for test_case in test_cases: slice_start = len(call_params) - test_case['skip'] - test_case['days'] slice_end = len(call_params) - test_case['skip'] call_slice = call_params[slice_start:slice_end] dry_run_slice = dry_run_params[slice_start:slice_end] calls = [call(call_param) for call_param in call_slice] prints = [call(' '.join(call_param)) for call_param in dry_run_slice] patch_call_path = 'feed_ingestion.bin.exec_flows.subprocess.call' patch_print_path = 'feed_ingestion.bin.exec_flows.log' with patch(patch_call_path) as mock_call, \ patch(patch_print_path) as mock_print: exec_flows.exec_multiple_days( 'foo', days=test_case['days'], skip=test_case['skip'], max_concurrent_flows=10) mock_call.assert_has_calls(calls) assert mock_call.call_count == test_case['days'] mock_print.assert_not_called() mock_call.reset_mock() mock_print.reset_mock() exec_flows.exec_multiple_days( 'foo', days=test_case['days'], skip=test_case['skip'], max_concurrent_flows=10, dry_run=True) mock_print.assert_has_calls(prints) assert mock_print.call_count == test_case['days'] mock_call.assert_not_called() @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') def test_exec_multiple_days_max_concurrent_flows_less_than_days( mock_count_running_workflows_by_type, mock_get_flow): """Test workflow execution when max_concurrent_flows is less than days.""" mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' days = 5 max_concurrent_flows = 3 with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: calls = [ call(['garcon', 'exec', 'foo', '--context', '{{"context_date": "2015-09-{:02d}"}}'.format(i)]) for i in range(26, 29)] exec_flows.exec_multiple_days( 'foo', days=days, skip=0, max_concurrent_flows=max_concurrent_flows) mock_call.assert_has_calls(calls) assert mock_call.call_count == max_concurrent_flows @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') def test_exec_multiple_days_workflows_running( mock_count_running_workflows_by_type, mock_get_flow): """Test workflow execution when previous workflows are running.""" mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' days = 5 mock_count_running_workflows_by_type.return_value = 3 with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: calls = [ call(['garcon', 'exec', 'foo', '--context', '{"context_date": "2015-09-26"}'])] exec_flows.exec_multiple_days( 'foo', days=days, skip=0, max_concurrent_flows=4) mock_call.assert_has_calls(calls) assert mock_call.call_count == 1 @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @patch('feed_ingestion.bin.exec_flows.garcon_feed_status._get_item') def test_exec_multiple_days_all_dates_ingested( mock_get_item, mock_count_running_workflows_by_type, mock_get_flow): """Test workflow execution when all dates have been processed.""" mock_get_item.return_value = { 'status': garcon_feed_status.STATUS_INGESTED } mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: exec_flows.exec_multiple_days( 'foo', days=5, skip=0, max_concurrent_flows=10, check_status=True) mock_call.assert_not_called() @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @patch('feed_ingestion.bin.exec_flows.garcon_feed_status._get_item') def test_exec_multiple_days_some_dates_ingested( mock_get_item, mock_count_running_workflows_by_type, mock_get_flow): """Test workflow execution when some dates have been processed.""" # mock feed status to return INGESTED for dates before 2015-09-28 def is_ingested(*args): if args[1] < '2015-09-28': return { 'status': garcon_feed_status.STATUS_INGESTED } return { 'status': garcon_feed_status.STATUS_NOT_AVAILABLE } mock_get_item.side_effect = is_ingested mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: # test remaining days are processed calls = [ call(['garcon', 'exec', 'foo', '--context', '{{"context_date": "2015-09-{:02d}"}}'.format(i)]) for i in range(28, 31)] exec_flows.exec_multiple_days( 'foo', days=5, skip=0, max_concurrent_flows=10, check_status=True) mock_call.assert_has_calls(calls) assert mock_call.call_count == 3 mock_call.reset_mock() # limit max_concurrent_flows to two calls = [ call(['garcon', 'exec', 'foo', '--context', '{{"context_date": "2015-09-{:02d}"}}'.format(i)]) for i in range(28, 30)] exec_flows.exec_multiple_days( 'foo', days=5, skip=0, max_concurrent_flows=2, check_status=True) mock_call.assert_has_calls(calls) assert mock_call.call_count == 2 @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') def test_pass_initial_context(mock_count_running_workflows_by_type, mock_get_flow): """Test initial context is passed to all calls to Garcon.""" mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' initial_context = '{"foo": "bar", "hello": "World"}' calls = [ call([ 'garcon', 'exec', 'foo', '--context', '{{"context_date": "2015-09-{:02d}", ' '"foo": "bar", "hello": "World"}}'.format(i)]) for i in range(28, 30) ] with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: exec_flows.exec_multiple_days( 'foo', days=2, skip=1, max_concurrent_flows=5, context=initial_context) mock_call.assert_has_calls(calls) assert mock_call.call_count == 2 @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @patch('feed_ingestion.bin.exec_flows.garcon_feed_status._get_item') @patch('feed_ingestion.bin.exec_flows.subprocess') def test_contextified_feed_name( mock_subprocess, mock_get_item, mock_count_running_workflows_by_type, mock_get_flow,): """Test execution if feed_name depends on context.""" class TestFlow(base.FlowBase): def contextified_feed_name(self, context): return context['custom_feed_name'] mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.return_value = TestFlow('test', 'foo') initial_context = ( '{"foo": "bar", "hello": "World",' '"custom_feed_name": "custom_foo"}') exec_flows.exec_multiple_days( 'foo', days=1, skip=1, check_status=True, max_concurrent_flows=5, context=initial_context) mock_get_item.assert_called_with('custom_foo', mock.ANY) @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @patch('feed_ingestion.bin.exec_flows.garcon_feed_status._get_item') def test_completed_1( mock_get_item, mock_count_running_workflows_by_type, mock_get_flow): # if STATUS_INGESTED and completed=False, it should not proceed. def is_ingested(*args): return { 'status': garcon_feed_status.STATUS_INGESTED, task_status.FIELD_COMPLETED: True } mock_get_item.side_effect = is_ingested mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: exec_flows.exec_multiple_days( 'foo', days=1, skip=0, max_concurrent_flows=10, check_status=True) assert mock_call.call_count == 0 @freeze_time('2015-09-30') @patch('feed_ingestion.bin.exec_flows.flows.get_flow') @patch('feed_ingestion.bin.exec_flows.swf_util.' 'count_running_workflows_by_type') @patch('feed_ingestion.bin.exec_flows.garcon_feed_status._get_item') def test_completed_2( mock_get_item, mock_count_running_workflows_by_type, mock_get_flow): # if STATUS_INGESTED and completed=False, it should proceed. def is_ingested(*args): return { 'status': garcon_feed_status.STATUS_INGESTED, task_status.FIELD_COMPLETED: False } mock_get_item.side_effect = is_ingested mock_count_running_workflows_by_type.return_value = 0 mock_get_flow.domain = 'test' mock_get_flow.name = 'foo' with patch('feed_ingestion.bin.exec_flows.subprocess.call') as mock_call: exec_flows.exec_multiple_days( 'foo', days=1, skip=0, max_concurrent_flows=10, check_status=True) assert mock_call.call_count == 1