"""Unit tests for Garcon tasks and utils for Feed Ingestion workflows.""" from unittest.mock import MagicMock from unittest.mock import patch from garcon import task from garcon_contrib.dynamo_feed_status import \ garcon_feed_status import pytest from feed_ingestion.tasks import check_ingested_status from feed_ingestion.tasks import check_status @patch('feed_ingestion.tasks.task_status') def test_check_status(task_status_mock): """Test check_status decorator. Normal circumstances. Activity and date are present and reside first in the argument list. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' task_id = 'my_task' date = '2017-10-20' @check_status(feed_name, task_id=task_id) def task(activity, date, a, b, c): return dict(a=a, b=b, c=c) result = task(MagicMock(), date, 'a', 'b', 'c') assert result == dict(a='a', b='b', c='c') task_status_mock.is_completed_task.assert_called_once_with( feed_name, date, task_id) task_status_mock.mark_completed_task.assert_called_once_with( feed_name, date, task_id) @patch('feed_ingestion.tasks.task_status') def test_check_status_already_completed(task_status_mock): """Test check_status decorator. The task has already been completed. Just return. """ task_status_mock.is_completed_task.return_value = True feed_name = 'test_feed' task_id = 'my_task' date = '2017-10-20' @check_status(feed_name, task_id=task_id) def task(activity, date, a, b, c): return dict(a=a, b=b, c=c) result = task(MagicMock(), date, 'a', 'b', 'c') assert result is None task_status_mock.is_completed_task.assert_called_once_with( feed_name, date, task_id) task_status_mock.mark_completed_task.assert_not_called() @patch('feed_ingestion.tasks.task_status') def test_check_status_changed_param_order(task_status_mock): """Test check_status decorator. Activity and date may reside anywhere in argument list. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' task_id = 'my_task' date = '2017-10-20' @check_status(feed_name, task_id=task_id) def task(a, activity, b, date, c): # non-default params order return dict(a=a, b=b, c=c) result = task('a', MagicMock(), 'b', date, 'c') assert result == dict(a='a', b='b', c='c') task_status_mock.is_completed_task.assert_called_once_with( feed_name, date, task_id) task_status_mock.mark_completed_task.assert_called_once_with( feed_name, date, task_id) @patch('feed_ingestion.tasks.task_status') def test_check_status_no_required_args(task_status_mock): """Test check_status decorator. Activity and date are required. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' task_id = 'my_task' @check_status(feed_name, task_id=task_id) def task_no_date(activity, a, b, c): return a, b, c @check_status(feed_name, task_id=task_id) def task_no_activity(date, a, b, c): return a, b, c with pytest.raises(Exception): task_no_date(MagicMock(), 'a', 'b', 'c') task_no_activity('2017-10-20', 'a', 'b', 'c') @patch('feed_ingestion.tasks.task_status') def test_check_status_no_task_id_specified(task_status_mock): """Test check_status decorator. Generate task_id. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' date = '2017-10-20' @check_status(feed_name=feed_name) def task(activity, date, a, b, c): return dict(a=a, b=b, c=c) result = task(MagicMock(), date, 'a', 'b', 'c') assert result == dict(a='a', b='b', c='c') task_status_mock.mark_completed_task.assert_called_once_with( feed_name, date, 'task') @patch('feed_ingestion.tasks.task_status') def test_check_status_no_feed_name_specified(task_status_mock): """Test check_status decorator. Resolve feed_name dynamically. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' task_id = 'my_task' date = '2017-10-20' @check_status(task_id=task_id) def task(activity, date, feed_name, a, b, c): return dict(feed_name=feed_name, a=a, b=b, c=c) result = task(MagicMock(), date, feed_name, 'a', 'b', 'c') assert result == dict(feed_name=feed_name, a='a', b='b', c='c') task_status_mock.mark_completed_task.assert_called_once_with( feed_name, date, task_id) @patch('feed_ingestion.tasks.task_status') def test_check_status_no_feed_name(task_status_mock): """Test check_status decorator. No feed_name specified (nor statically nor as a call param). """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' date = '2017-10-20' @check_status() def task(activity, date, a, b, c): return feed_name, a, b, c with pytest.raises(Exception): task(MagicMock(), date, 'a', 'b', 'c') @patch('feed_ingestion.tasks.task_status') def test_check_status_garcon(task_status_mock): """Test check_status decorator. Use garcon's task.decorate. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' date = '2017-10-20' context = { 'a': 'a', 'b': 'b', 'c': 'c', 'date': date, } @task.decorate(timeout=300) @check_status(feed_name=feed_name) def task_garcon(activity, date, a, b, c): return dict( date=date, a=a + '_UPD', b=b + '_UPD', c=c + '_UPD', d='outstanding d' ) task_filled = task_garcon.fill( date='date', a='a', b='b', c='c' ) assert task_filled(context, activity=MagicMock()) == { 'a': 'a_UPD', 'b': 'b_UPD', 'c': 'c_UPD', 'd': 'outstanding d', 'date': date, } task_status_mock.mark_completed_task.assert_called_once_with( feed_name, date, 'task_garcon') @patch('feed_ingestion.tasks.task_status') def test_check_status_garcon_raises_exception(task_status_mock): """Test check_status when the underlying func raises an exception. Use garcon's task.decorate. """ task_status_mock.is_completed_task.return_value = False feed_name = 'test_feed' date = '2017-10-20' context = { 'a': 'a', 'b': 'b', 'c': 'c', 'date': date, } @task.decorate(timeout=300) @check_status(feed_name=feed_name) def task_garcon(activity, date, a, b, c): raise Exception('How about this?') task_filled = task_garcon.fill( date='date', a='a', b='b', c='c' ) with pytest.raises(Exception): task_filled(context, activity=MagicMock()) task_status_mock.mark_completed_task.assert_not_called() @patch('feed_ingestion.tasks.garcon_feed_status') def test_check_ingested_status(feed_status_mock): """Test check_ingested_status decorator. Normal circumstances. Activity is present and reside first in the argument list. """ feed_status_mock.get_overall_status.return_value = ( garcon_feed_status.STATUS_NOT_INGESTED) feed_name = 'test_feed' date = '2017-10-20' @check_ingested_status(feed_name, skip_check=False) def task(activity, date, a, b, c): return a, b, c result = task(MagicMock(), date, 'a', 'b', 'c') assert result == ('a', 'b', 'c') @patch('feed_ingestion.tasks.garcon_feed_status') def test_check_ingested_status_already_completed(feed_status_mock): """Test check_ingested_status decorator. The task has already been completed. Return the stop response. """ feed_status_mock.STATUS_INGESTED = garcon_feed_status.STATUS_INGESTED feed_status_mock.get_overall_status.return_value = ( garcon_feed_status.STATUS_INGESTED) feed_name = 'test_feed' date = '2017-10-20' @check_ingested_status(feed_name, skip_check=False) def task(activity, date, a, b, c): return a, b, c result = task(MagicMock(), date, 'a', 'b', 'c') assert result == {'stop': True} @patch('feed_ingestion.tasks.garcon_feed_status') def test_check_ingested_status_changed_param_order(feed_status_mock): """Test check_ingested_status decorator. Activity and date may reside anywhere in argument list. """ feed_status_mock.get_overall_status.return_value = ( garcon_feed_status.STATUS_NOT_INGESTED) feed_name = 'test_feed' date = '2017-10-20' @check_ingested_status(feed_name, skip_check=False) def task(activity, b, date, a, c): return a, b, c result = task(MagicMock(), 'b', date, 'a', 'c') assert result == ('a', 'b', 'c') @patch('feed_ingestion.tasks.garcon_feed_status') def test_check_ingested_status_no_feed_name_specified(feed_status_mock): """Test check_ingested_status decorator. Date is required. """ feed_status_mock.get_overall_status.return_value = ( garcon_feed_status.STATUS_NOT_INGESTED) feed_name = 'test_feed' @check_ingested_status(feed_name, skip_check=False) def task(activity, b, a, c): return a, b, c with pytest.raises(Exception): task(MagicMock(), 'b', 'a', 'c') @patch('feed_ingestion.tasks.garcon_feed_status') def test_check_ingested_status_no_required_args(feed_status_mock): """Test check_ingested_status decorator. Resolve feed_name dynamically. """ feed_status_mock.get_overall_status.return_value = ( garcon_feed_status.STATUS_NOT_INGESTED) feed_name = 'test_feed' date = '2017-10-20' @check_ingested_status(skip_check=False) def task(activity, feed_name, date, a, b): return feed_name, a, b result = task(MagicMock(), feed_name, date, 'a', 'b') assert result == (feed_name, 'a', 'b') @patch('feed_ingestion.tasks.garcon_feed_status') def test_check_ingested_status_no_feed_name(feed_status_mock): """Test check_ingested_status decorator. No feed_name specified (nor statically nor as a call param). """ feed_status_mock.get_overall_status.return_value = ( garcon_feed_status.STATUS_NOT_INGESTED) date = '2017-10-20' @check_ingested_status(skip_check=False) def task(activity, date, b, a, c): return a, b, c with pytest.raises(Exception): task(MagicMock(), date, 'b', 'a', 'c')