"""Unit tests for tasks for Apple's Reporter Tool.""" from unittest.mock import call, MagicMock, patch import boto3 from garcon import activity from garcon_contrib.dynamo_feed_status import \ garcon_feed_status import pytest from feed_ingestion.tasks import reporter_tasks from feed_ingestion.tasks.reporter_tasks import _handle_file_name_incorrection from feed_ingestion.util.itunes_reporter import ReporterException @patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status') @patch('feed_ingestion.tasks.reporter_tasks.s3') @patch('feed_ingestion.tasks.reporter_tasks.os') @patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter') @patch('feed_ingestion.tasks.reporter_tasks._get_cred') @patch('feed_ingestion.util.task_status.add_newcontext') @patch('feed_ingestion.util.task_status.update_report_context_status') @patch('feed_ingestion.util.task_status.get_report_contexts') def test_extract_reporter_file_to_s3( mock_get_report_contexts, mock_update_context_status, mock_add_new_context, mock__get_cred, mock_itunes_reporter, mock_os, mock_s3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning): """Test extract_reporter_file_to_s3.""" # setup mocks mock_reporter = MagicMock() file_name = 'file01.txt.gz' reporter_return = \ '' \ '' \ 'Successfully downloaded {0}' \ ' ' \ .format(file_name) mock_reporter.get_sales_report_file_name.return_value = file_name mock_reporter.download_sales_report.return_value = reporter_return mock_itunes_reporter.get_reporter.return_value = mock_reporter mock_s3.upload_to_s3.return_value = 188 reporter_account = 'ORCHARD' report_type = 'amStreams' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' # test successful file download with feed_status calls context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) # assert reporter calls mock_itunes_reporter.get_reporter.assert_called_with( reporter_account, date, None, None, access_token=mock__get_cred.return_value) mock_reporter.get_sales_report_file_name.assert_called_with(report_type) mock_reporter.download_sales_report.assert_called_with(report_type) # assert s3/file calls assert mock_s3.upload_to_s3.call_args_list == [ call( file_path='file01.txt.gz', bucket_name='foo', object_key='bar/file01.txt.gz', )] mock_os.remove.assert_called_with(file_name) mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_DOWNLOADED) # assert context assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED} # test zip uploading zip_name = file_name.replace('.txt.gz', '.zip') def check_name(name): return name == zip_name mock_os.path.exists = MagicMock(side_effect=check_name) with patch('feed_ingestion.tasks.reporter_tasks.ZipFile') as mock_zip: nl = ['file1', 'file2', 'file3'] mock_zip.return_value.__enter__.return_value.namelist.return_value = nl mock_set_feed_status.reset_mock() context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) # test file not available mock_reporter.download_sales_report = MagicMock( side_effect=ReporterException( 'message', ReporterException.FILE_UNAVAILABLE_ERR_CODE)) context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE} # test feed_status not called if feed name not passed mock_set_feed_status.reset_mock() context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path) assert mock_set_feed_status.call_count == 0 # check if file is not uploaded on s3 context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE} # test feed_status not downloaded if file fails to upload in s3 (inc. zip) with patch( 'feed_ingestion.tasks.reporter_tasks._upload_to_s3', side_effect=Exception): mock_set_feed_status.reset_mock() context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) assert context != {file_name: garcon_feed_status.STATUS_DOWNLOADED} # test exception raised if not ReporterException.FILE_UNAVAILABLE_ERR_CODE mock_reporter.download_sales_report = MagicMock( side_effect=ReporterException('message', 0)) context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path) mock_reporter.download_sales_report = MagicMock( side_effect=Exception('message')) assert mock_send_error_or_warning.call_count == 1 assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE} with pytest.raises(Exception): context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path) assert mock_add_new_context.called @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status') @patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter') @patch('feed_ingestion.tasks.reporter_tasks._get_cred') def test_extract_reporter_file_to_s3_already_downloaded( mock__get_cred, mock_itunes_reporter, mock_feed_get_status): """Test extract_reporter_file_to_s3 if feed status is DOWNLOADED.""" # setup mocks mock_reporter = MagicMock() file_name = 'file01' mock_reporter.get_sales_report_file_name.return_value = file_name mock_itunes_reporter.get_reporter.return_value = mock_reporter reporter_account = 'ORCHARD' report_type = 'amStreams' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' # test successful file download with feed_status calls mock_feed_get_status.return_value = garcon_feed_status.STATUS_DOWNLOADED context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) # assert reporter calls mock_itunes_reporter.get_reporter.assert_called_with( reporter_account, date, None, None, access_token=mock__get_cred.return_value ) mock_reporter.get_sales_report_file_name.assert_called_with(report_type) mock_reporter.download_sales_report.call_count == 0 # assert context assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED} @patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status') @patch('feed_ingestion.tasks.reporter_tasks.s3') @patch('feed_ingestion.tasks.reporter_tasks.os') @patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter') @patch('feed_ingestion.tasks.reporter_tasks._get_cred') @patch('feed_ingestion.util.task_status.add_newcontext') @patch('feed_ingestion.util.task_status.update_report_context_status') @patch('feed_ingestion.util.task_status.get_report_contexts') def test_extract_reporter_file_to_s3_finance( mock_get_report_contexts, mock_update_context_status, mock_add_new_context, mock__get_cred, mock_itunes_reporter, mock_os, mock_s3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning): """Test extract_reporter_file_to_s3 for finance reports.""" # setup mocks mock_reporter = MagicMock() file_name = 'file01' reporter_return = \ '' \ '' \ 'Successfully downloaded {0}' \ ' ' \ .format(file_name) mock_reporter.get_finance_report_file_name.return_value = file_name mock_reporter.download_finance_report.return_value = reporter_return mock_itunes_reporter.get_reporter.return_value = mock_reporter reporter_account = 'ORCHARD' report_type = 'AppleMusicSummary' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' # test successful file download with feed_status calls context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name, report_role='finance', report_country='UA') # assert reporter calls mock_itunes_reporter.get_reporter.assert_called_with( reporter_account, date, 'UA', None, access_token=mock__get_cred.return_value ) mock_reporter.get_finance_report_file_name.assert_called_with(report_type) mock_reporter.download_finance_report.assert_called_with(report_type) # assert s3/file calls assert mock_s3.upload_to_s3.call_args_list == [ call( file_path=file_name, bucket_name='foo', object_key='bar/file01', )] mock_os.remove.assert_called_with(file_name) mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_DOWNLOADED) # assert context assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED} # test feed_status not called if feed name not passed mock_set_feed_status.reset_mock() context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, report_role='finance', report_country='UA') assert mock_set_feed_status.call_count == 0 # test file not available mock_reporter.download_finance_report = MagicMock( side_effect=ReporterException( 'message', ReporterException.FILE_UNAVAILABLE_ERR_CODE)) context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name, report_role='finance', report_country='UA') mock_set_feed_status.assert_called_with( feed_name, date, file_name, status=garcon_feed_status.STATUS_NOT_AVAILABLE) assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE} # test feed_status not called if feed name not passed mock_set_feed_status.reset_mock() context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, report_role='finance', report_country='UA') assert mock_set_feed_status.call_count == 0 # test exception raised if not ReporterException.FILE_UNAVAILABLE_ERR_CODE mock_reporter.download_finance_report = MagicMock( side_effect=ReporterException('message', 0)) context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, report_role='finance', report_country='UA') assert mock_send_error_or_warning.call_count == 1 assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE} mock_reporter.download_finance_report = MagicMock( side_effect=Exception('message')) with pytest.raises(Exception): context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, report_role='finance', report_country='UA') @patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status') @patch('feed_ingestion.tasks.reporter_tasks.s3') @patch('feed_ingestion.tasks.reporter_tasks.os') @patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter') @patch('feed_ingestion.tasks.reporter_tasks._get_cred') @patch('feed_ingestion.util.task_status.add_newcontext') @patch('feed_ingestion.util.task_status.update_report_context_status') @patch('feed_ingestion.util.task_status.get_report_contexts') def test_file_rename_in_reporter_file_to_s3( mock_get_report_contexts, mock_update_context_status, mock_add_new_context, mock__get_cred, mock_itunes_reporter, mock_os, mock_s3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning): """Test extract_reporter_file_to_s3.""" # setup mocks mock_reporter = MagicMock() file_name = 'file01.txt.gz' reporter_file_name = 'file01_[1-2].txt.gz' reporter_return = \ '' \ '' \ 'Successfully downloaded {0}' \ ' ' \ .format(reporter_file_name) mock_reporter.get_sales_report_file_name.return_value = file_name mock_reporter.download_sales_report.return_value = reporter_return mock_itunes_reporter.get_reporter.return_value = mock_reporter reporter_account = 'ORCHARD' report_type = 'amStreams' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' # test successful file download with feed_status calls reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) # assert reporter calls assert mock_s3.upload_to_s3.call_args_list == [ call( file_path='file01.txt.gz', bucket_name='foo', object_key='bar/file01.txt.gz', )] mock_os.rename.assert_called_with(reporter_file_name, file_name) @patch('feed_ingestion.tasks.reporter_tasks.is_zipfile') @patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status') @patch('feed_ingestion.tasks.reporter_tasks.s3') @patch('feed_ingestion.tasks.reporter_tasks.os') @patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter') @patch('feed_ingestion.tasks.reporter_tasks._get_cred') @patch('feed_ingestion.util.task_status.add_newcontext') @patch('feed_ingestion.util.task_status.update_report_context_status') @patch('feed_ingestion.util.task_status.get_report_contexts') def test_extract_reporter_file_to_s3_zip_disguised_as_gz( mock_get_report_contexts, mock_update_context_status, mock_add_new_context, mock__get_cred, mock_itunes_reporter, mock_os, mock_s3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning, mock_is_zipfile): """Apple-delivered .txt.gz that is really ZIP gets renamed & extracted.""" # setup mocks mock_reporter = MagicMock() file_name = 'AppleMusic_SongUniques_80028967_20260513.txt.gz' zip_name = 'AppleMusic_SongUniques_80028967_20260513.zip' reporter_return = \ '' \ '' \ 'Successfully downloaded {0}' \ ' ' \ .format(file_name) mock_reporter.get_sales_report_file_name.return_value = file_name mock_reporter.download_sales_report.return_value = reporter_return mock_itunes_reporter.get_reporter.return_value = mock_reporter mock_s3.upload_to_s3.return_value = 188 # the local file lives at .txt.gz pre-rename and at .zip post-rename mock_is_zipfile.return_value = True existing_files = {file_name} def fake_rename(src, dst): existing_files.discard(src) existing_files.add(dst) mock_os.rename.side_effect = fake_rename mock_os.path.exists = MagicMock( side_effect=lambda name: name in existing_files) reporter_account = 'ORCHARD' report_type = 'amSongs' date = '2026-05-13' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' with patch('feed_ingestion.tasks.reporter_tasks.ZipFile') as mock_zip: extracted = ['AppleMusic_SongUniques_80028967_20260513.txt'] mock_zip.return_value.__enter__.return_value.namelist.return_value = ( extracted) context = reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) # the .txt.gz file was detected as ZIP and renamed mock_is_zipfile.assert_called_with(file_name) mock_os.rename.assert_any_call(file_name, zip_name) # the extracted contents were uploaded under bar/ assert mock_s3.upload_to_s3.call_args_list == [ call( file_path=extracted[0], bucket_name='foo', object_key='bar/{}'.format(extracted[0]), )] assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED} @patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status') @patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status') @patch('feed_ingestion.tasks.reporter_tasks.s3') @patch('feed_ingestion.tasks.reporter_tasks.os') @patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter') @patch('feed_ingestion.tasks.reporter_tasks._get_cred') @patch('feed_ingestion.util.task_status.add_newcontext') @patch('feed_ingestion.util.task_status.update_report_context_status') @patch('feed_ingestion.util.task_status.get_report_contexts') def test_file_rename_different_versions_in_reporter_file_to_s3( mock_get_report_contexts, mock_update_context_status, mock_add_new_context, mock__get_cred, mock_itunes_reporter, mock_os, mock_s3, mock_set_feed_status, mock_get_feed_status, mock_send_error_or_warning): """Test extract_reporter_file_to_s3.""" # setup mocks mock_reporter = MagicMock() file_name = 'file01_V1_0.txt.gz' reporter_file_name = 'file01_V1_1.txt.gz' reporter_return = \ '' \ '' \ 'Successfully downloaded {0}' \ ' ' \ .format(reporter_file_name) mock_reporter.get_sales_report_file_name.return_value = file_name mock_reporter.download_sales_report.return_value = reporter_return mock_itunes_reporter.get_reporter.return_value = mock_reporter reporter_account = 'ORCHARD' report_type = 'amStreams' date = '2016-01-01' destination_s3_path = 's3://foo/bar/' feed_name = 'Feed' # test successful file download with feed_status calls reporter_tasks.extract_reporter_file_to_s3( activity.Activity(boto3.client('swf', 'us-east-1')), reporter_account, report_type, date, destination_s3_path, feed_name) # assert reporter calls # ensure the file was not renamed by _handle_file_name_incorrection assert mock_os.rename.call_count == 0 assert mock_s3.upload_to_s3.call_args_list == [ call( file_path='file01_V1_0.txt.gz', bucket_name='foo', object_key='bar/file01_V1_0.txt.gz', )] @patch.object(reporter_tasks, 'SWFSecretsManager') @pytest.mark.parametrize( 'cred_name, is_optional, expected', [ ('existing_cred', True, 'value'), ('existing_cred', False, 'value'), ('not_existing_cred', True, None), ('not_existing_cred', False, reporter_tasks.ClientError), ] ) def test__get_cred(mock_secrets_manager, cred_name, is_optional, expected): """Test _get_cred.""" def get_cred_side_effect(name): if name == 'existing_cred': return 'value' raise reporter_tasks.ClientError( error_response={'Error': {'Code': 'ResourceNotFoundException'}}, operation_name='Some' ) mock_secrets_manager.return_value.get_cred.side_effect = ( get_cred_side_effect) try: result = reporter_tasks._get_cred( cred_name=cred_name, optional=is_optional) assert result == expected except Exception as e: assert isinstance(e, expected) assert mock_secrets_manager.call_args_list == [ call(environment='dev', service_name='itunesconnect') ] @patch.object(reporter_tasks, '_get_cred') @pytest.mark.parametrize( 'licensor, expected_value', [ ('existing_licensor', 'value_licensor'), ('not_existing_licensor', 'value_common'), ] ) def test__get_access_token(mock_get_cred, licensor, expected_value): """Test _get_access_token.""" def get_cred_side_effect(cred_name, optional): creds = { 'AccessToken_existing_licensor': 'value_licensor', 'AccessToken': 'value_common', } return creds.get(cred_name) mock_get_cred.side_effect = get_cred_side_effect result = reporter_tasks._get_access_token( licensor=licensor) assert result == expected_value @patch('feed_ingestion.tasks.reporter_tasks.os') @pytest.mark.parametrize( 'reporter_response_filename, expected_filename, file_is_renamed', [ ('AppleMusic_SongUniques_80028967_20230801_V1_0_[1-2].txt.gz', 'AppleMusic_SongUniques_80028967_20230801_V1_0.txt.gz', True), ('AppleMusic_SongUniques_80028967_20230801_V1_1.zip', 'AppleMusic_SongUniques_80028967_20230801_V1_0.txt.gz', False), ('AppleMusic_LibraryEvents_80012345_20210101.txt.gz', 'AppleMusic_LibraryEvents_80012345_20210101_V5_0.txt.gz', False), ('AppleMusic_LibraryEvents_80012345_20210101_[5-2].txt.gz', 'AppleMusic_LibraryEvents_80012345_20210101_V5_0.txt.gz', False), ('AppleMusic_LibraryEvents_80012345_20210101_V5_0_[5-2].txt.gz', 'AppleMusic_LibraryEvents_80012345_20210101_V5_0.txt.gz', True), ('iTunes_ContentDemographics_80012345_20170320_V1_2_[1-2].zip', 'iTunes_ContentDemographics_80012345_20170320_V1_2.zip', True), ('AppleMusic_SummaryStreams_80012345_20210101_V1_1.zip', ' AppleMusic_SummaryStreams_80012345_20210101_V1_2.zip', False), # incorrect behaviour ('AppleMusic_SongUniques_80028967_20241014_V1_0_[1-2].txt.gz', 'AppleMusic_SongUniques_80028967_20241014.txt.gz', True), ] ) def test__handle_file_name_incorrection(mock_os, reporter_response_filename, expected_filename, file_is_renamed): """Test _handle_file_name_incorrection.""" reporter_resp = f""" {reporter_response_filename} """ mock_activity = MagicMock() _handle_file_name_incorrection(reporter_resp, expected_filename, mock_activity) if file_is_renamed: mock_os.rename.assert_called_with(reporter_response_filename, expected_filename) else: assert mock_os.rename.call_count == 0