"""Tests for Proper New Releases Tracks flow's tasks.""" from unittest.mock import MagicMock from unittest.mock import patch from botocore.exceptions import ClientError from freezegun import freeze_time import pytest from feed_sender.flows import tasks as base_tasks from feed_sender.flows.proper_new_releases_tracks import tasks from feed_sender.flows.proper_new_releases_tracks.conf import settings @freeze_time('2016-01-01 09:50:00') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.bootstrap_util') def test_bootstrap_reload_false(mock_bootstrap_util): """Test bootstrap with reload False.""" mock_activity = MagicMock() mock_activity.logger.info.return_value = None # Testing reload when feed is already sent. mock_bootstrap_util.feed_already_sent.return_value = True resp = tasks.bootstrap(mock_activity, '2016-01-01 09:50:00', False) assert resp == {'stop': True, 'message': ( 'Feed for 2016-01-01 has already been sent.')} # Testing reload when feed is has not yet been sent. mock_bootstrap_util.feed_already_sent.return_value = False resp = tasks.bootstrap(mock_activity, '2016-01-01 09:50:00', False) expected = settings.NEW_RELEASES_TRACKLISTS_S3_PATH.format( date='2016-01-01') assert resp == { 'context_date': '2016-01-01', 'feed_name': 'proper_new_releases_tracks', 'cutoff': '2016-01-01 09:50:00', 'new_releases_tracklistings_s3_path': expected, 'new_releases_filename': 'essential_setups_2016_01_01.csv', 'new_tracklistings_filename': ( 'essential_setups_tracklistings_2016_01_01.csv'), 'new_releases_tracklistings_sftp_path': settings.SFTP_REMOTE_PATH, 'release_ids_filename': 'release_ids_2016-01-01.txt', 'filenames_for_sftp_activity': [ 'essential_setups_2016_01_01.csv', 'essential_setups_tracklistings_2016_01_01.csv' ], 'job_ids_filename': 'job_ids_2016-01-01.txt' } @freeze_time('2016-01-01 09:50:00') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.bootstrap_util') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.helpers') def test_bootstrap_reload_true(mock_helpers, mock_bootstrap_util): """Test bootstrap with reload True.""" feed_name = 'proper_new_releases_tracks' context_date = '2016-01-01' cutoff = '2016-01-01 09:50:00' mock_activity = MagicMock() mock_helpers.delete_status.return_value = None mock_activity.logger.info.return_value = None # Testing reload when feed is already sent. mock_bootstrap_util.feed_already_sent.return_value = True resp = tasks.bootstrap(mock_activity, None, True) assert resp == {'stop': True, 'message': ( 'Feed for 2016-01-01 has already been sent.')} # Testing reload when feed is has not yet been sent. mock_bootstrap_util.feed_already_sent.return_value = False resp = tasks.bootstrap(mock_activity, cutoff, True) expected = settings.NEW_RELEASES_TRACKLISTS_S3_PATH.format( date='2016-01-01') assert resp == { 'context_date': context_date, 'feed_name': 'proper_new_releases_tracks', 'cutoff': cutoff, 'new_releases_tracklistings_s3_path': expected, 'new_releases_filename': 'essential_setups_2016_01_01.csv', 'new_tracklistings_filename': ( 'essential_setups_tracklistings_2016_01_01.csv'), 'new_releases_tracklistings_sftp_path': settings.SFTP_REMOTE_PATH, 'release_ids_filename': 'release_ids_2016-01-01.txt', 'filenames_for_sftp_activity': [ 'essential_setups_2016_01_01.csv', 'essential_setups_tracklistings_2016_01_01.csv' ], 'job_ids_filename': 'job_ids_2016-01-01.txt' } mock_helpers.delete_status.assert_any_call(feed_name, context_date) @freeze_time('2016-01-01 09:50:00') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.bootstrap_util') def test_bootstrap_reload_false_no_context_date(mock_bootstrap_util): """Test bootstrap with reload True and no context_date.""" mock_activity = MagicMock() mock_activity.logger.info.return_value = None mock_bootstrap_util.feed_already_sent.return_value = True resp = tasks.bootstrap(mock_activity, None, False) assert resp == {'stop': True, 'message': ( 'Feed for 2016-01-01 has already been sent.')} mock_bootstrap_util.feed_already_sent.return_value = False resp = tasks.bootstrap(mock_activity, cutoff=None, reload=False) expected = settings.NEW_RELEASES_TRACKLISTS_S3_PATH.format( date='2016-01-01') assert resp.get('context_date') == '2016-01-01' assert resp.get('new_releases_tracklistings_s3_path') == expected @freeze_time('2016-01-01 09:50:00') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.bootstrap_util') def test_bootstrap_no_cutoff(mock_bootstrap_util): """Test bootstrap with no cutoff. (For a feed not sent and reload is False). """ cutoff = '2016-01-01 09:50:00' mock_activity = MagicMock() mock_activity.logger.info.return_value = None # Feed has not yet been sent. mock_bootstrap_util.feed_already_sent.return_value = False resp = tasks.bootstrap(mock_activity, reload=False) expected = settings.NEW_RELEASES_TRACKLISTS_S3_PATH.format( date='2016-01-01') assert resp == { 'context_date': '2016-01-01', 'feed_name': 'proper_new_releases_tracks', 'cutoff': cutoff, 'new_releases_tracklistings_s3_path': expected, 'new_releases_filename': 'essential_setups_2016_01_01.csv', 'new_tracklistings_filename': ( 'essential_setups_tracklistings_2016_01_01.csv'), 'new_releases_tracklistings_sftp_path': settings.SFTP_REMOTE_PATH, 'release_ids_filename': 'release_ids_2016-01-01.txt', 'filenames_for_sftp_activity': [ 'essential_setups_2016_01_01.csv', 'essential_setups_tracklistings_2016_01_01.csv' ], 'job_ids_filename': 'job_ids_2016-01-01.txt' } @patch('feed_sender.flows.tasks.helpers') def test_set_status(mock_helpers): """Test set_status.""" mock_helpers.STATUS_SENT = 'SENT' mock_helpers.STATUS_UPLOADED_TO_S3 = 'UPLOADED_TO_S3' mock_activity = MagicMock() mock_activity.logger.info.return_value = None base_tasks.set_status( mock_activity, 'proper_new_releases_tracks', '2016-01-01', 'SENT') mock_helpers.delete_status.return_value = None mock_helpers.set_status.assert_any_call( 'proper_new_releases_tracks', '2016-01-01', 'SENT') mock_activity.logger.info.assert_any_call( 'Workflow proper_new_releases_tracks on 2016-01-01 has completed.') @patch('feed_sender.flows.tasks.helpers') def test_set_status_not_complete_yet(mock_helpers): """Test set_status not complete yet.""" mock_helpers.STATUS_SENT = 'SENT' mock_helpers.STATUS_UPLOADED_TO_S3 = 'UPLOADED_TO_S3' mock_helpers.delete_status.return_value = None mock_activity = MagicMock() mock_activity.logger.info.return_value = None base_tasks.set_status( mock_activity, 'ProperNewReleasesTracks', '2016-01-01', 'PROCESSING') assert not mock_activity.logger.info.called @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper') def test_generate_feed_for_releases_past_threshold(mock_proper): """Test when release ids exceed threshold.""" mock_proper.return_value = mock_proper mock_proper.total_releases = 5 mock_proper.releases_threshold = 3 mock_proper.close_sql.return_value = None mock_proper.filter_products_query.return_value = None mock_proper.save_release_ids_to_s3.return_value = True mock_proper.convert_releases_to_csv.return_value = True mock_activity = MagicMock() mock_activity.logger.info.return_value = None tasks.generate_feed_for_releases( mock_activity, '2000-06-30', '2000-06-30 09:50', 's3://test', 'abc.csv', 'release_ids.txt', False, 'job_ids.txt') mock_activity.logger.info.assert_any_call( 'Total Releases are: 5') mock_activity.logger.warning.assert_any_call( '5 is greater than release threshold 3') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper', return_value=MagicMock(total_releases=3, releases_threshold=10)) def test_generate_feed_for_releases_success(mock_proper): """Test success for task.""" mock_activity = MagicMock() context = tasks.generate_feed_for_releases( mock_activity, '2000-06-30', '2000-06-30 09:50:00', 's3://test', 'abc.csv', 'release_ids.txt', False, 'job_ids.txt') assert context == {'success': True} mock_proper.assert_called_with('2000-06-30 09:50:00') assert mock_proper.return_value.connect_to_sql.called mock_proper.return_value.convert_releases_to_csv.assert_called_with( 's3://test', 'abc.csv', False) assert mock_proper.return_value.close_sql.called @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper') def test_generate_feed_for_tracks_success(mock_proper): """Test success for task.""" mock_activity = MagicMock() context = tasks.generate_feed_for_tracks( mock_activity, '2000-06-30 09:50:00', 's3://test', 'abc.csv', 'release_ids.txt') assert context == {'success': True} mock_proper.assert_called_with('2000-06-30 09:50:00') assert mock_proper.return_value.connect_to_sql.called mock_proper.return_value.convert_tracks_to_csv.assert_called_with( 's3://test', 'abc.csv', 'release_ids.txt', False) assert mock_proper.return_value.close_sql.called @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper') def test_generate_feed_for_tracks_failed(mock_proper): """Test failure for task.""" mock_activity = MagicMock() mock_proper.return_value.convert_tracks_to_csv.side_effect = Exception() with pytest.raises(Exception): tasks.generate_feed_for_tracks( mock_activity, '2000-06-30 09:50:00', 's3://test', 'abc.csv', 'release_ids.txt') assert mock_proper.return_value.close_sql.called @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper') def test_update_delivery_history_fail(mock_proper): """Test for error in db write.""" mock_proper.return_value = mock_proper mock_proper.physical_delivery_db = 'history_db' mock_proper.connect_to_sql.return_value = MagicMock() mock_proper.close_sql.return_value = None mock_proper.update_delivery_history.return_value = False mock_activity = MagicMock() mock_activity.logger.info.return_value = None resp = tasks.update_delivery_history( mock_activity, '2000-06-30 09:50', 's3://test', 'abc.csv') assert 'stop' in resp assert resp.get('message') == 'Error in writing to history_db' @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper') def test_update_delivery_history_success(mock_proper): """Test for success in updating DB.""" mock_proper.return_value = mock_proper mock_proper.connect_to_sql.return_value = MagicMock() mock_proper.close_sql.return_value = None mock_proper.update_delivery_history.return_value = True mock_activity = MagicMock() mock_activity.logger.info.return_value = None resp = tasks.update_delivery_history( mock_activity, '2000-06-30 09:50', 's3://test', 'abc.csv') assert 'success' in resp @patch('feed_sender.flows.proper_new_releases_tracks.tasks.proper_common') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.job_status.update') def test_update_status_from_job_ids_file( mock_job_status_sql_call, mock_proper_common): """Test if _job_ids are updated in database.""" mock_job_status_sql_call.return_value = True mock_proper_common.get_ids_from_s3.return_value = { 'release_ids': [11111, 22222, 33333], 'job_ids': [1, 2, 3]} update_result = tasks.update_job_status( MagicMock(), 's3://abc', 'file.csv', 'encoding') assert mock_job_status_sql_call.call_count == 3 assert update_result == {'success': True} @patch('feed_sender.flows.proper_new_releases_tracks.tasks.proper_common') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.job_status.update') def test_update_status_failed_from_job_ids_file( mock_job_status_update, mock_proper_common): """Test if _job_ids are updated in database.""" mock_proper_common.get_ids_from_s3.return_value = { 'release_ids': [], 'job_ids': []} update_result = tasks.update_job_status( MagicMock(), 's3://abc', 'file.csv', 'encoding') assert not mock_job_status_update.called assert update_result == {'success': True} @patch('feed_sender.flows.proper_new_releases_tracks.tasks.proper_common') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.Proper') def test_generate_feed_for_releases_in_vector(mock_proper, mock_proper_common): """Test when proper_in_vector is True.""" mock_proper.return_value = mock_proper mock_proper.total_releases = 5 mock_proper.releases_threshold = 10 mock_proper.fetch_products.return_value = { 'release_ids': '123,332', 'job_ids': '345,2345'} mock_proper.save_release_ids_to_s3.return_value = True mock_proper.convert_releases_to_csv.return_value = True mock_proper_common.save_job_ids_to_s3.return_value = None mock_activity = MagicMock() mock_activity.logger.info.return_value = None response = tasks.generate_feed_for_releases( mock_activity, '2000-06-30', '2000-06-30 09:50', 's3://test', 'abc.csv', 'release_ids.txt', True, 'job_ids.txt') assert mock_proper_common.save_job_ids_to_s3.called_with( 's3://test', 'job_ids.txt', '345,2345') assert not mock_proper.filter_products_query.called assert 'success' in response @patch('feed_sender.flows.proper_new_releases_tracks.tasks.mysql') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.proper_common') def test_obtain_jobs(mock_proper_common, mock_mysql): """Test when there's messages in the SQS Queue.""" mock_mysql.return_value = MagicMock() mock_proper_common.get_products_from_queue.return_value = { 'release_ids': [1, 2, 3], 'job_ids': [11111, 22222, 33333]} tasks.obtain_jobs(MagicMock(), 'mock_s3_path', 'mock_job_ids_filename') mock_proper_common.save_job_ids_to_s3.assert_called_with( 'mock_s3_path', 'mock_job_ids_filename', [11111, 22222, 33333]) @patch('feed_sender.flows.proper_new_releases_tracks.tasks.mysql') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.proper_common') def test_obtain_jobs_no_jobs(mock_proper_common, mock_mysql): """Test when there's messages in the SQS Queue.""" mock_mysql.get_dd_db_connection_pymysql.return_value = MagicMock() mock_proper_common.get_products_from_queue.return_value = None result = tasks.obtain_jobs(MagicMock(), MagicMock(), MagicMock()) assert not mock_proper_common.save_job_ids_to_s3.called assert result == {'error': 'No jobs available from queue.'} @patch('feed_sender.flows.proper_new_releases_tracks.tasks.mysql') @patch('feed_sender.flows.proper_new_releases_tracks.tasks.proper_common') def test_obtain_jobs_error_queue(mock_proper_common, mock_mysql): """Test when there's messages in the SQS Queue.""" mock_mysql.get_dd_db_connection_pymysql.return_value = MagicMock() mock_proper_common.get_products_from_queue.side_effect = ClientError( {'Error': {'Code': 1, 'Message': 'Mock Client Error'}}, 'MockOp') result = tasks.obtain_jobs(MagicMock(), MagicMock(), MagicMock()) assert not mock_proper_common.save_job_ids_to_s3.called assert result == { 'error': 'An error occurred (1) when calling the MockOp operation: ' 'Mock Client Error'}