from collections import deque from unittest.mock import MagicMock, patch, call from freezegun import freeze_time from garcon_contrib.dynamo_feed_status import \ garcon_feed_status as feed_status import pytest def test_get_attribute_name_for_file_status(): date = '2015-08-12' file_name = 'AppleMusic_Event_2015-08-12.txt' expected_result = 'AppleMusic_Event_status' actual_result = feed_status.get_attribute_name_for_file_status( file_name, date) assert expected_result == actual_result @pytest.fixture() def mock_config(): with patch.object(feed_status, 'config') as mock_config: mock_config.feed_ingestion_table = 'd_table' yield mock_config @patch.object(feed_status, 'dynamodb') def test_set_missing_files(mock_dynamodb_client, mock_config): feed_name = 'feed' date = '2015-08-12' missing_files = [ 's3://some_path/somefile1.txt', 's3://some_path/somefile2.txt'] feed_status.set_missing_files(feed_name, date, missing_files) assert mock_dynamodb_client.update_item.call_args_list == [ call( TableName='d_table', Key={'feed_name': {'S': 'feed'}, 'date': {'S': '2015-08-12'}}, UpdateExpression='SET missing_files = :missing_files', ExpressionAttributeValues={ ':missing_files': { 'S': 's3://some_path/somefile1.txt,' 's3://some_path/somefile2.txt'}}, ), ] @patch.object(feed_status, 'dynamodb') def test_set_status(mock_dynamodb_client, mock_config): feed_name = 'feed' date = '2015-08-12' file_name = 'AppleMusic_Event_2015-08-12.txt' status = feed_status.STATUS_NOT_INGESTED feed_status.set_status(feed_name, date, file_name, status) assert mock_dynamodb_client.update_item.call_args_list == [ call( TableName='d_table', Key={'feed_name': {'S': 'feed'}, 'date': {'S': '2015-08-12'}}, UpdateExpression='SET #file_name_status = :file_name_status', ExpressionAttributeNames={ '#file_name_status': 'AppleMusic_Event_status'}, ExpressionAttributeValues={ ':file_name_status': {'S': 'NOT_INGESTED'}}, ReturnValues='NONE', ), ] @patch.object(feed_status, 'dynamodb') @patch.object(feed_status, '_determine_overall_status') @pytest.mark.parametrize( 'kwargs, expected_status', [ (dict(), 'INGESTED'), (dict(overall_status=feed_status.STATUS_NOT_INGESTED), 'NOT_INGESTED'), ] ) def test_set_overall_status( mock__determine_overall_status, mock_dynamodb_client, mock_config, kwargs, expected_status): feed_name = 'feed' date = '2015-08-12' mock__determine_overall_status.return_value = feed_status.STATUS_INGESTED feed_status.set_overall_status(feed_name, date, **kwargs) assert mock_dynamodb_client.update_item.call_args_list == [ call( TableName='d_table', Key={'date': {'S': '2015-08-12'}, 'feed_name': {'S': 'feed'}}, UpdateExpression='SET #status = :status', ExpressionAttributeNames={'#status': 'status'}, ExpressionAttributeValues={':status': {'S': expected_status}}, ), ] @patch.object(feed_status, 'dynamodb') @patch.object(feed_status, '_determine_overall_status') def test_set_overall_status_with_attributes( mock__determine_overall_status, mock_dynamodb_client, mock_config): feed_name = 'feed' date = '2015-08-12' mock__determine_overall_status.return_value = feed_status.STATUS_INGESTED feed_status.set_overall_status( feed_name, date, attributes={'counter': 123, 'some': 'value'}) assert mock_dynamodb_client.update_item.call_args_list == [ call( TableName='d_table', Key={'feed_name': {'S': 'feed'}, 'date': {'S': '2015-08-12'}}, UpdateExpression=('SET #counter = :counter' ', #some = :some' ', #status = :status'), ExpressionAttributeNames={ '#counter': 'counter', '#some': 'some', '#status': 'status'}, ExpressionAttributeValues={ ':counter': {'N': '123'}, ':some': {'S': 'value'}, ':status': {'S': 'INGESTED'}}) ] @pytest.mark.parametrize( 'get_value_return_value, expected_result', [ (False, False), ({'status': 'INGESTED'}, 'INGESTED'), ] ) def test_get_overall_status( get_value_return_value, expected_result, monkeypatch): """ Test getting status of feed workflow """ feed_name = 'feed' date = '2015-08-12' # Test when overall status IN dynamo monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=get_value_return_value)) actual_result = feed_status.get_overall_status(feed_name, date) assert actual_result == expected_result @pytest.mark.parametrize( 'file_name, get_value_return_value, expected_result', [ ('AppleMusic_Event_2015-08-12.txt', False, None), ('AppleMusic_Event_2015-08-12.txt', {'AppleMusic_Event_status': feed_status.STATUS_DOWNLOADED}, 'DOWNLOADED'), ('AppleMusic_Event.txt', {'AppleMusic_Event_status': feed_status.STATUS_DOWNLOADED}, 'DOWNLOADED'), ] ) def test_get_status( file_name, get_value_return_value, expected_result, monkeypatch): """ Test getting status of file for a feed workflow """ feed_name = 'feed' date = '2015-08-12' monkeypatch.setattr( feed_status, '_get_item', lambda a, b: get_value_return_value) actual_result = feed_status.get_status(feed_name, date, file_name) assert actual_result == expected_result monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=False)) actual_result = feed_status.get_status(feed_name, date, file_name) assert actual_result is None @pytest.mark.parametrize( 'case, get_item_return_value, expected_result', [ ( 'files not available so overall status is STATUS_NOT_AVAILABLE', { 'AppleMusic_Ev_80028967_status': feed_status.STATUS_NOT_AVAILABLE, 'AppleMusic_Ev_80029727_status': feed_status.STATUS_NOT_AVAILABLE, 'date': '2015-07-08', 'feed_name': 'Apple Music', 'status': feed_status.STATUS_NOT_INGESTED }, feed_status.STATUS_NOT_AVAILABLE, ), ( 'files in mixed state, so status is STATUS_NOT_INGESTED', { 'AppleMusic_Ev_80028967_status': feed_status.STATUS_NOT_AVAILABLE, 'AppleMusic_Ev_80029727_status': feed_status.STATUS_DOWNLOADED, 'date': '2015-07-08', 'feed_name': 'Apple Music', 'status': feed_status.STATUS_NOT_INGESTED, }, feed_status.STATUS_NOT_INGESTED, ), ( 'files all downloaded so overall status is STATUS_DOWNLOADED', { 'AppleMusic_Ev_80028967_status': feed_status.STATUS_DOWNLOADED, 'AppleMusic_Ev_80029727_status': feed_status.STATUS_DOWNLOADED, 'date': '2015-07-08', 'feed_name': 'Apple Music', 'status': feed_status.STATUS_NOT_INGESTED, }, feed_status.STATUS_DOWNLOADED, ), ] ) def test_determine_overall_status( case, get_item_return_value, expected_result, monkeypatch): """ Test getting overall status of feed workflow based on files downloaded Note, _determine_overall_statu assumes there is always at least one file status for the flow in dynamo """ feed_name = 'Apple Music' date = '20150708' monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=get_item_return_value)) actual_result = feed_status._determine_overall_status(feed_name, date) assert actual_result == expected_result def test_determine_overall_status_negative(monkeypatch): feed_name = 'Apple Music' date = '20150708' with pytest.raises(AssertionError): # No files downloaded, no status available in the status table monkeypatch.setattr( feed_status, '_get_item', MagicMock(return_value=False)) feed_status._determine_overall_status(feed_name, date) @pytest.mark.parametrize( 'get_item_return_value, expected_result', [ ( {'Item': {'status': {'S': 'INGESTED'}}}, {'status': 'INGESTED'}, ), ( {'Item': {'status': {'S': 'INGESTED'}, 'file': {'S': '1111'}}}, {'status': 'INGESTED', 'file': '1111'}, ), ( {'Item': {'status': {'S': 'INGESTED'}, 'counter': {'N': 123}}}, {'status': 'INGESTED', 'counter': 123}, ), ({'Other': 1235}, None), ({}, None), ] ) @patch.object(feed_status, 'dynamodb') def test_get_item( mock_dynamodb, get_item_return_value, expected_result, mock_config): mock_dynamodb.get_item.return_value = get_item_return_value returned = feed_status._get_item('feed_name', '2015-08-01') assert returned == expected_result assert mock_dynamodb.get_item.call_args_list == [ call( TableName='d_table', Key={'feed_name': {'S': 'feed_name'}, 'date': {'S': '2015-08-01'}} ) ] @freeze_time('2016-02-10') def test_get_missing_dates(monkeypatch): monkeypatch.setattr( feed_status, 'get_overall_status', MagicMock(return_value='NOT_AVAILABLE')) actual = feed_status.get_missing_dates('iTunes', 3) expected = deque(['2016-02-07', '2016-02-08', '2016-02-09', '2016-02-10']) assert expected == actual