"""Unit tests for tasks for Apple's Reporter Tool.""" from unittest.mock import MagicMock, patch import boto3 from garcon import activity from garcon_contrib.dynamo_feed_status import \ garcon_feed_status import pytest from feed_ingestion.tasks import apple_podcasts_reporter_tasks from feed_ingestion.util.itunes_reporter import ReporterException @pytest.fixture def mock_os(): """Yield task status.""" path = 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.os' with patch(path) as os_mock: os_mock.path.join.return_value = './current_dir' yield os_mock @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'send_error_or_warning') @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'garcon_feed_status.get_status') @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'garcon_feed_status.set_status') @patch('feed_ingestion.tasks.apple_podcasts_reporter_tasks.boto3') @patch('feed_ingestion.tasks.apple_podcasts_reporter_tasks.itunes_reporter') def test_extract_reporter_file_to_s3( mock_itunes_reporter, mock_boto3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning, mock_os): """Test extract_reporter_file_to_s3.""" reporter_account = 1234 vendor = 5678 file_name = 'ApplePodcasts_SalesSummary_5678_20160101.txt.gz' report_type = 'apSalesSummary' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' secrets_path = 'secrets-path' # setup mocks mock_reporter = MagicMock() mock_reporter.get_sales_report_file_name.return_value = file_name mock_itunes_reporter.get_apple_podcast_reporter.return_value = \ mock_reporter mock_key = MagicMock() mock_os.path.join.return_value = \ './ApplePodcasts_SalesSummary_5678_20160101.txt.gz' mock_upload_file = MagicMock() mock_boto3.client.return_value.upload_file.return_value = mock_upload_file mock_head_object = MagicMock() mock_boto3.client.return_value.head_object.return_value = mock_head_object mock_key.exists.return_value = True # test successful file download with feed_status calls context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name) # assert reporter calls mock_itunes_reporter.get_apple_podcast_reporter.assert_called_with( reporter_account, vendor, date, secrets_path) mock_reporter.get_sales_report_file_name.assert_called_with(report_type) mock_reporter.download_sales_report.assert_called_with(report_type) # assert s3/file calls mock_boto3.client.return_value.upload_file.assert_called_with( './ApplePodcasts_SalesSummary_5678_20160101.txt.gz', 'foo', 'bar/ApplePodcasts_SalesSummary_5678_20160101.txt.gz') mock_boto3.client.return_value.head_object.assert_called_with( Bucket='foo', Key='bar/ApplePodcasts_SalesSummary_5678_20160101.txt.gz') mock_os.remove.assert_called_with(file_name) mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_DOWNLOADED) # assert context assert context == { 'file_name': file_name, 'status': garcon_feed_status.STATUS_DOWNLOADED } # test zip uploading zip_name = file_name.replace('.txt.gz', '.zip') def check_name(name): return name == zip_name mock_os.path.exists = MagicMock(side_effect=check_name) with patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.ZipFile' ) as mock_zip: nl = ['file1', 'file2', 'file3'] mock_zip.return_value.__enter__.return_value.namelist.return_value = nl mock_set_feed_status.reset_mock() context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name) # test file not available mock_reporter.download_sales_report = MagicMock( side_effect=ReporterException( 'message', ReporterException.FILE_UNAVAILABLE_ERR_CODE)) context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name) mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) assert context == { 'file_name': file_name, 'status': garcon_feed_status.STATUS_NOT_AVAILABLE } # test feed_status not called if feed name not passed mock_set_feed_status.reset_mock() context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path) assert mock_set_feed_status.call_count == 0 # check if file is not uploaded on s3 mock_key.exists.return_value = False context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name) assert context == { 'file_name': file_name, 'status': garcon_feed_status.STATUS_NOT_AVAILABLE } mock_key.exists.return_value = True # test feed_status not downloaded if file fails to upload in s3 (inc. zip) with patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks._upload_to_s3', side_effect=Exception): mock_set_feed_status.reset_mock() context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name) assert context != {file_name: garcon_feed_status.STATUS_DOWNLOADED} # test exception raised if not ReporterException.FILE_UNAVAILABLE_ERR_CODE mock_reporter.download_sales_report = MagicMock( side_effect=ReporterException('message', 0)) context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path) mock_reporter.download_sales_report = MagicMock( side_effect=Exception('message')) assert mock_send_error_or_warning.call_count == 1 assert context == { 'file_name': file_name, 'status': garcon_feed_status.STATUS_NOT_AVAILABLE } with pytest.raises(Exception): context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path) @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'send_error_or_warning') @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'garcon_feed_status.get_status') @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'garcon_feed_status.set_status') @patch('feed_ingestion.tasks.apple_podcasts_reporter_tasks.boto3') @patch('feed_ingestion.tasks.apple_podcasts_reporter_tasks.itunes_reporter') def test_extract_reporter_file_to_s3_for_monthly_report( mock_itunes_reporter, mock_boto3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning, mock_os): """Test extract_reporter_file_to_s3.""" reporter_account = 1234 vendor = 5678 file_name = 'ApplePodcasts_SalesSummary_Monthly_5678_20160101.txt.gz' report_type = 'apSalesSummary' date = '2016-01-01' date_type = 'Monthly' etl_name = 'apple_podcasts_sales_summary_monthly' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' secrets_path = 'secrets-path' # setup mocks mock_reporter = MagicMock() mock_reporter.get_sales_report_file_name.return_value = file_name mock_itunes_reporter.get_apple_podcast_reporter.return_value = \ mock_reporter mock_key = MagicMock() mock_os.path.join.return_value = \ './ApplePodcasts_SalesSummary_Monthly_5678_20160101.txt.gz' mock_upload_file = MagicMock() mock_boto3.client.return_value.upload_file.return_value = mock_upload_file mock_head_object = MagicMock() mock_boto3.client.return_value.head_object.return_value = mock_head_object mock_key.exists.return_value = True # test successful file download with feed_status calls context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name, date_type=date_type, etl_name=etl_name) # assert reporter calls expected_report_type = f'{report_type}{date_type}' mock_itunes_reporter.get_apple_podcast_reporter.assert_called_with( reporter_account, vendor, date, secrets_path) mock_reporter.get_sales_report_file_name.assert_called_with( expected_report_type) mock_reporter.download_sales_report.assert_called_with( expected_report_type) # assert s3/file calls mock_boto3.client.return_value.upload_file.assert_called_with( './ApplePodcasts_SalesSummary_Monthly_5678_20160101.txt.gz', 'foo', 'bar/ApplePodcasts_SalesSummary_Monthly_5678_20160101.txt.gz') mock_boto3.client.return_value.head_object.assert_called_with( Bucket='foo', Key='bar/ApplePodcasts_SalesSummary_Monthly_5678_20160101.txt.gz') mock_os.remove.assert_called_with(file_name) mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_DOWNLOADED) # assert context assert context == { 'file_name': file_name, 'status': garcon_feed_status.STATUS_DOWNLOADED } @patch( 'feed_ingestion.tasks.apple_podcasts_reporter_tasks.' 'garcon_feed_status.get_status') @patch('feed_ingestion.tasks.apple_podcasts_reporter_tasks.itunes_reporter') def test_extract_reporter_file_to_s3_already_downloaded( mock_itunes_reporter, mock_feed_get_status): """Test extract_reporter_file_to_s3 if feed status is DOWNLOADED.""" reporter_account = 1234 vendor = 5678 file_name = 'ApplePodcasts_SalesSummary_5678_20160101.txt.gz' report_type = 'apSalesSummary' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' secrets_path = 'secrets-path' # setup mocks mock_reporter = MagicMock() mock_reporter.get_sales_report_file_name.return_value = file_name mock_itunes_reporter.get_apple_podcast_reporter.return_value = \ mock_reporter # test successful file download with feed_status calls mock_feed_get_status.return_value = garcon_feed_status.STATUS_DOWNLOADED context = apple_podcasts_reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, vendor, date, secrets_path, destination_s3_path, feed_name) # assert reporter calls mock_itunes_reporter.get_apple_podcast_reporter.assert_called_with( reporter_account, vendor, date, secrets_path) mock_reporter.get_sales_report_file_name.assert_called_with(report_type) mock_reporter.download_sales_report.call_count == 0 # assert context assert context == { 'file_name': file_name, 'status': garcon_feed_status.STATUS_DOWNLOADED }