from collections import deque from unittest.mock import MagicMock from unittest.mock import patch from boto.dynamodb2.table import Table from freezegun import freeze_time from garcon.contrib.dynamo_feed_status import \ feed_status_ingestion as feed_status import pytest @pytest.yield_fixture def mock_dynamodb_connection(): """Yield DynamoDB connection.""" dynamodb_connection_path = ( 'garcon.contrib.dynamo_feed_status.feed_status_ingestion.' 'get_dynamodb_connection') with patch(dynamodb_connection_path) as dynamodb_connection: yield dynamodb_connection @patch('garcon.contrib.dynamo_feed_status.feed_status_ingestion.' 'DynamoDBConnection') def test_get_dynamodb_connection(conn_mock): """Test get_dynamodb_connection function.""" mm = MagicMock() conn_mock.return_value = mm conn = feed_status.get_dynamodb_connection() assert conn is mm @patch('garcon.contrib.dynamo_feed_status.feed_status_ingestion.Table') def test_get_dynamodb_table(table_mock, mock_dynamodb_connection): """Test get_dynamodb_table function.""" mm = MagicMock() table_mock.return_value = mm table = feed_status.get_dynamodb_table() assert table is mm assert mock_dynamodb_connection.called @patch('garcon.contrib.dynamo_feed_status.feed_status_ingestion.Table') def test_get_dynamodb_table_with_conn(table_mock, mock_dynamodb_connection): mm = MagicMock() con = MagicMock() table_mock.return_value = mm table = feed_status.get_dynamodb_table(con) assert not mock_dynamodb_connection.called assert table is mm def test_get_attribute_name_for_file_status(): date = '2015-08-12' file_name = 'AppleMusic_Event_2015-08-12.txt' expected_result = 'AppleMusic_Event_status' actual_result = feed_status.get_attribute_name_for_file_status( file_name, date) assert expected_result == actual_result def test_set_status(monkeypatch, mock_dynamodb_connection): feed_name = 'feed' date = '2015-08-12' file_name = 'AppleMusic_Event_2015-08-12.txt' status = feed_status.STATUS_NOT_INGESTED monkeypatch.setattr( feed_status, 'DYNAMODB_TABLE', 'd_table') feed_status.set_status(feed_name, date, file_name, status) mock_dynamodb_connection.return_value.update_item.assert_called_with( 'd_table', {'feed_name': {'S': 'feed'}, 'date': {'S': '2015-08-12'}}, expression_attribute_names={ '#file_name_status': 'AppleMusic_Event_status'}, expression_attribute_values={ ':file_name_status': { 'S': '{}'.format(feed_status.STATUS_NOT_INGESTED)}}, update_expression='SET #file_name_status = ' ':file_name_status') def test_set_overall_status(monkeypatch, mock_dynamodb_connection): feed_name = 'feed' date = '2015-08-12' monkeypatch.setattr( feed_status, 'DYNAMODB_TABLE', 'd_table') monkeypatch.setattr( feed_status, '_determine_overall_status', MagicMock( return_value=feed_status.STATUS_INGESTED)) # Test when overall status NOT explicitly pass feed_status.set_overall_status(feed_name, date) mock_dynamodb_connection.return_value.update_item.assert_called_with( 'd_table', {'feed_name': {'S': 'feed'}, 'date': {'S': '2015-08-12'}}, expression_attribute_names={'#status': 'status'}, expression_attribute_values={ ':status': {'S': '{}'.format(feed_status.STATUS_INGESTED)}}, update_expression='SET #status = :status') # Test when overall status IS explicitly pass feed_status.set_overall_status( feed_name, date, overall_status=feed_status.STATUS_NOT_INGESTED) mock_dynamodb_connection.return_value.update_item.assert_called_with( 'd_table', {'feed_name': {'S': 'feed'}, 'date': {'S': '2015-08-12'}}, expression_attribute_names={'#status': 'status'}, expression_attribute_values={ ':status': {'S': '{}'.format(feed_status.STATUS_NOT_INGESTED)}}, update_expression='SET #status = :status') def test_get_overall_status(monkeypatch): """ Test getting status of feed workflow """ feed_name = 'feed' date = '2015-08-12' # Test when overall status IN dynamo monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=dict(status=feed_status.STATUS_INGESTED))) actual_result = feed_status.get_overall_status(feed_name, date) expected_result = feed_status.STATUS_INGESTED assert actual_result == expected_result # Test when overall status NOT IN dynamo monkeypatch.setattr( feed_status, '_get_item', MagicMock(return_value=False)) actual_result = feed_status.get_overall_status(feed_name, date) expected_result = False assert actual_result == expected_result def test_get_status(monkeypatch): """ Test getting status of file for a feed workflow """ feed_name = 'feed' date = '2015-08-12' file_name = 'AppleMusic_Event_2015-08-12.txt' expected_result = 'AppleMusic_Event_status' monkeypatch.setattr( feed_status, '_get_item', MagicMock(return_value=dict( AppleMusic_Event_status=feed_status.STATUS_DOWNLOADED))) monkeypatch.setattr( feed_status, 'get_attribute_name_for_file_status', MagicMock( return_value='AppleMusic_Event_status')) actual_result = feed_status.get_status(feed_name, date, file_name) expected_result = feed_status.STATUS_DOWNLOADED assert actual_result == expected_result def test_determine_overall_status(monkeypatch): """ Test getting overall status of feed workflow based on files downloaded Note, _determine_overall_statu assumes there is always at least one file status for the flow in dynamo """ feed_name = 'Apple Music' date = '20150708' # files not available so overal status is STATUS_NOT_AVAILABLE expected_result = feed_status.STATUS_NOT_AVAILABLE monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=dict( AppleMusic_Ev_80028967_status=feed_status.STATUS_NOT_AVAILABLE, AppleMusic_Ev_80029727_status=feed_status.STATUS_NOT_AVAILABLE, date=date, feed_name=feed_name, status=feed_status.STATUS_NOT_INGESTED))) actual_result = feed_status._determine_overall_status(feed_name, date) assert actual_result == expected_result # files in mixed state, so status is STATUS_NOT_INGESTED expected_result = feed_status.STATUS_NOT_INGESTED monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=dict( AppleMusic_Ev_80028967_status=feed_status.STATUS_NOT_AVAILABLE, AppleMusic_Ev_80029727_status=feed_status.STATUS_DOWNLOADED, date=date, feed_name=feed_name, status=feed_status.STATUS_NOT_INGESTED))) actual_result = feed_status._determine_overall_status(feed_name, date) assert actual_result == expected_result # files all downloaded so overall status is STATUS_DOWNLOADED expected_result = feed_status.STATUS_DOWNLOADED monkeypatch.setattr( feed_status, '_get_item', MagicMock( return_value=dict( AppleMusic_Event_80028967_status=feed_status.STATUS_DOWNLOADED, AppleMusic_Event_80029727_status=feed_status.STATUS_DOWNLOADED, date=date, feed_name=feed_name, status=feed_status.STATUS_NOT_INGESTED))) actual_result = feed_status._determine_overall_status(feed_name, date) assert actual_result == expected_result with pytest.raises(AssertionError): # No files downloaded, no status available in the status table monkeypatch.setattr( feed_status, '_get_item', MagicMock(return_value=False)) feed_status._determine_overall_status(feed_name, date) def test_get_item(monkeypatch): mock_item = dict( status=feed_status.STATUS_DOWNLOADED ) monkeypatch.setattr( Table, 'get_item', MagicMock(return_value=mock_item)) feed_status._get_item('feed_name', '2015-08-01') @freeze_time('2016-02-10') def test_get_missing_dates(monkeypatch): monkeypatch.setattr( feed_status, 'get_overall_status', MagicMock(return_value='NOT_AVAILABLE')) actual = feed_status.get_missing_dates('iTunes', 3) expected = deque(['2016-02-07', '2016-02-08', '2016-02-09', '2016-02-10']) assert expected == actual