"""Unit tests for tasks for Apple's Reporter Tool."""
from unittest.mock import call, MagicMock, patch
import boto3
from garcon import activity
from garcon_contrib.dynamo_feed_status import \
garcon_feed_status
import pytest
from feed_ingestion.tasks import reporter_tasks
from feed_ingestion.tasks.reporter_tasks import _handle_file_name_incorrection
from feed_ingestion.util.itunes_reporter import ReporterException
@patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status')
@patch('feed_ingestion.tasks.reporter_tasks.s3')
@patch('feed_ingestion.tasks.reporter_tasks.os')
@patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter')
@patch('feed_ingestion.tasks.reporter_tasks._get_cred')
@patch('feed_ingestion.util.task_status.add_newcontext')
@patch('feed_ingestion.util.task_status.update_report_context_status')
@patch('feed_ingestion.util.task_status.get_report_contexts')
def test_extract_reporter_file_to_s3(
mock_get_report_contexts,
mock_update_context_status,
mock_add_new_context,
mock__get_cred,
mock_itunes_reporter, mock_os, mock_s3,
mock_set_feed_status, mock_get_feed_status,
mock_send_error_or_warning):
"""Test extract_reporter_file_to_s3."""
# setup mocks
mock_reporter = MagicMock()
file_name = 'file01.txt.gz'
reporter_return = \
'' \
'' \
.format(file_name)
mock_reporter.get_sales_report_file_name.return_value = file_name
mock_reporter.download_sales_report.return_value = reporter_return
mock_itunes_reporter.get_reporter.return_value = mock_reporter
mock_s3.upload_to_s3.return_value = 188
reporter_account = 'ORCHARD'
report_type = 'amStreams'
date = '2016-01-01'
destination_s3_path = 's3://foo/bar/'
feed_name = 'Feed'
# test successful file download with feed_status calls
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
# assert reporter calls
mock_itunes_reporter.get_reporter.assert_called_with(
reporter_account, date, None, None,
access_token=mock__get_cred.return_value)
mock_reporter.get_sales_report_file_name.assert_called_with(report_type)
mock_reporter.download_sales_report.assert_called_with(report_type)
# assert s3/file calls
assert mock_s3.upload_to_s3.call_args_list == [
call(
file_path='file01.txt.gz',
bucket_name='foo',
object_key='bar/file01.txt.gz',
)]
mock_os.remove.assert_called_with(file_name)
mock_set_feed_status.assert_called_with(
feed_name, date, file_name,
status=garcon_feed_status.STATUS_DOWNLOADED)
# assert context
assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED}
# test zip uploading
zip_name = file_name.replace('.txt.gz', '.zip')
def check_name(name):
return name == zip_name
mock_os.path.exists = MagicMock(side_effect=check_name)
with patch('feed_ingestion.tasks.reporter_tasks.ZipFile') as mock_zip:
nl = ['file1', 'file2', 'file3']
mock_zip.return_value.__enter__.return_value.namelist.return_value = nl
mock_set_feed_status.reset_mock()
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
# test file not available
mock_reporter.download_sales_report = MagicMock(
side_effect=ReporterException(
'message', ReporterException.FILE_UNAVAILABLE_ERR_CODE))
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
mock_set_feed_status.assert_called_with(
feed_name, date, file_name,
status=garcon_feed_status.STATUS_NOT_AVAILABLE)
assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE}
# test feed_status not called if feed name not passed
mock_set_feed_status.reset_mock()
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path)
assert mock_set_feed_status.call_count == 0
# check if file is not uploaded on s3
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE}
# test feed_status not downloaded if file fails to upload in s3 (inc. zip)
with patch(
'feed_ingestion.tasks.reporter_tasks._upload_to_s3',
side_effect=Exception):
mock_set_feed_status.reset_mock()
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
assert context != {file_name: garcon_feed_status.STATUS_DOWNLOADED}
# test exception raised if not ReporterException.FILE_UNAVAILABLE_ERR_CODE
mock_reporter.download_sales_report = MagicMock(
side_effect=ReporterException('message', 0))
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path)
mock_reporter.download_sales_report = MagicMock(
side_effect=Exception('message'))
assert mock_send_error_or_warning.call_count == 1
assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE}
with pytest.raises(Exception):
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path)
assert mock_add_new_context.called
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status')
@patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter')
@patch('feed_ingestion.tasks.reporter_tasks._get_cred')
def test_extract_reporter_file_to_s3_already_downloaded(
mock__get_cred,
mock_itunes_reporter, mock_feed_get_status):
"""Test extract_reporter_file_to_s3 if feed status is DOWNLOADED."""
# setup mocks
mock_reporter = MagicMock()
file_name = 'file01'
mock_reporter.get_sales_report_file_name.return_value = file_name
mock_itunes_reporter.get_reporter.return_value = mock_reporter
reporter_account = 'ORCHARD'
report_type = 'amStreams'
date = '2016-01-01'
destination_s3_path = 's3://foo/bar/'
feed_name = 'Feed'
# test successful file download with feed_status calls
mock_feed_get_status.return_value = garcon_feed_status.STATUS_DOWNLOADED
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
# assert reporter calls
mock_itunes_reporter.get_reporter.assert_called_with(
reporter_account, date, None, None,
access_token=mock__get_cred.return_value
)
mock_reporter.get_sales_report_file_name.assert_called_with(report_type)
mock_reporter.download_sales_report.call_count == 0
# assert context
assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED}
@patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status')
@patch('feed_ingestion.tasks.reporter_tasks.s3')
@patch('feed_ingestion.tasks.reporter_tasks.os')
@patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter')
@patch('feed_ingestion.tasks.reporter_tasks._get_cred')
@patch('feed_ingestion.util.task_status.add_newcontext')
@patch('feed_ingestion.util.task_status.update_report_context_status')
@patch('feed_ingestion.util.task_status.get_report_contexts')
def test_extract_reporter_file_to_s3_finance(
mock_get_report_contexts,
mock_update_context_status,
mock_add_new_context,
mock__get_cred,
mock_itunes_reporter, mock_os, mock_s3,
mock_set_feed_status, mock_get_feed_status,
mock_send_error_or_warning):
"""Test extract_reporter_file_to_s3 for finance reports."""
# setup mocks
mock_reporter = MagicMock()
file_name = 'file01'
reporter_return = \
'' \
'' \
.format(file_name)
mock_reporter.get_finance_report_file_name.return_value = file_name
mock_reporter.download_finance_report.return_value = reporter_return
mock_itunes_reporter.get_reporter.return_value = mock_reporter
reporter_account = 'ORCHARD'
report_type = 'AppleMusicSummary'
date = '2016-01-01'
destination_s3_path = 's3://foo/bar/'
feed_name = 'Feed'
# test successful file download with feed_status calls
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name,
report_role='finance', report_country='UA')
# assert reporter calls
mock_itunes_reporter.get_reporter.assert_called_with(
reporter_account, date, 'UA', None,
access_token=mock__get_cred.return_value
)
mock_reporter.get_finance_report_file_name.assert_called_with(report_type)
mock_reporter.download_finance_report.assert_called_with(report_type)
# assert s3/file calls
assert mock_s3.upload_to_s3.call_args_list == [
call(
file_path=file_name,
bucket_name='foo',
object_key='bar/file01',
)]
mock_os.remove.assert_called_with(file_name)
mock_set_feed_status.assert_called_with(
feed_name, date, file_name,
status=garcon_feed_status.STATUS_DOWNLOADED)
# assert context
assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED}
# test feed_status not called if feed name not passed
mock_set_feed_status.reset_mock()
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, report_role='finance', report_country='UA')
assert mock_set_feed_status.call_count == 0
# test file not available
mock_reporter.download_finance_report = MagicMock(
side_effect=ReporterException(
'message', ReporterException.FILE_UNAVAILABLE_ERR_CODE))
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name,
report_role='finance', report_country='UA')
mock_set_feed_status.assert_called_with(
feed_name, date, file_name,
status=garcon_feed_status.STATUS_NOT_AVAILABLE)
assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE}
# test feed_status not called if feed name not passed
mock_set_feed_status.reset_mock()
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, report_role='finance', report_country='UA')
assert mock_set_feed_status.call_count == 0
# test exception raised if not ReporterException.FILE_UNAVAILABLE_ERR_CODE
mock_reporter.download_finance_report = MagicMock(
side_effect=ReporterException('message', 0))
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, report_role='finance', report_country='UA')
assert mock_send_error_or_warning.call_count == 1
assert context == {file_name: garcon_feed_status.STATUS_NOT_AVAILABLE}
mock_reporter.download_finance_report = MagicMock(
side_effect=Exception('message'))
with pytest.raises(Exception):
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, report_role='finance', report_country='UA')
@patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status')
@patch('feed_ingestion.tasks.reporter_tasks.s3')
@patch('feed_ingestion.tasks.reporter_tasks.os')
@patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter')
@patch('feed_ingestion.tasks.reporter_tasks._get_cred')
@patch('feed_ingestion.util.task_status.add_newcontext')
@patch('feed_ingestion.util.task_status.update_report_context_status')
@patch('feed_ingestion.util.task_status.get_report_contexts')
def test_file_rename_in_reporter_file_to_s3(
mock_get_report_contexts,
mock_update_context_status,
mock_add_new_context,
mock__get_cred,
mock_itunes_reporter, mock_os, mock_s3,
mock_set_feed_status, mock_get_feed_status,
mock_send_error_or_warning):
"""Test extract_reporter_file_to_s3."""
# setup mocks
mock_reporter = MagicMock()
file_name = 'file01.txt.gz'
reporter_file_name = 'file01_[1-2].txt.gz'
reporter_return = \
'' \
'' \
.format(reporter_file_name)
mock_reporter.get_sales_report_file_name.return_value = file_name
mock_reporter.download_sales_report.return_value = reporter_return
mock_itunes_reporter.get_reporter.return_value = mock_reporter
reporter_account = 'ORCHARD'
report_type = 'amStreams'
date = '2016-01-01'
destination_s3_path = 's3://foo/bar/'
feed_name = 'Feed'
# test successful file download with feed_status calls
reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
# assert reporter calls
assert mock_s3.upload_to_s3.call_args_list == [
call(
file_path='file01.txt.gz',
bucket_name='foo',
object_key='bar/file01.txt.gz',
)]
mock_os.rename.assert_called_with(reporter_file_name, file_name)
@patch('feed_ingestion.tasks.reporter_tasks.is_zipfile')
@patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status')
@patch('feed_ingestion.tasks.reporter_tasks.s3')
@patch('feed_ingestion.tasks.reporter_tasks.os')
@patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter')
@patch('feed_ingestion.tasks.reporter_tasks._get_cred')
@patch('feed_ingestion.util.task_status.add_newcontext')
@patch('feed_ingestion.util.task_status.update_report_context_status')
@patch('feed_ingestion.util.task_status.get_report_contexts')
def test_extract_reporter_file_to_s3_zip_disguised_as_gz(
mock_get_report_contexts,
mock_update_context_status,
mock_add_new_context,
mock__get_cred,
mock_itunes_reporter, mock_os, mock_s3,
mock_set_feed_status, mock_get_feed_status,
mock_send_error_or_warning,
mock_is_zipfile):
"""Apple-delivered .txt.gz that is really ZIP gets renamed & extracted."""
# setup mocks
mock_reporter = MagicMock()
file_name = 'AppleMusic_SongUniques_80028967_20260513.txt.gz'
zip_name = 'AppleMusic_SongUniques_80028967_20260513.zip'
reporter_return = \
'' \
'' \
.format(file_name)
mock_reporter.get_sales_report_file_name.return_value = file_name
mock_reporter.download_sales_report.return_value = reporter_return
mock_itunes_reporter.get_reporter.return_value = mock_reporter
mock_s3.upload_to_s3.return_value = 188
# the local file lives at .txt.gz pre-rename and at .zip post-rename
mock_is_zipfile.return_value = True
existing_files = {file_name}
def fake_rename(src, dst):
existing_files.discard(src)
existing_files.add(dst)
mock_os.rename.side_effect = fake_rename
mock_os.path.exists = MagicMock(
side_effect=lambda name: name in existing_files)
reporter_account = 'ORCHARD'
report_type = 'amSongs'
date = '2026-05-13'
destination_s3_path = 's3://foo/bar/'
feed_name = 'Feed'
with patch('feed_ingestion.tasks.reporter_tasks.ZipFile') as mock_zip:
extracted = ['AppleMusic_SongUniques_80028967_20260513.txt']
mock_zip.return_value.__enter__.return_value.namelist.return_value = (
extracted)
context = reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
# the .txt.gz file was detected as ZIP and renamed
mock_is_zipfile.assert_called_with(file_name)
mock_os.rename.assert_any_call(file_name, zip_name)
# the extracted contents were uploaded under bar/
assert mock_s3.upload_to_s3.call_args_list == [
call(
file_path=extracted[0],
bucket_name='foo',
object_key='bar/{}'.format(extracted[0]),
)]
assert context == {file_name: garcon_feed_status.STATUS_DOWNLOADED}
@patch('feed_ingestion.tasks.reporter_tasks.send_error_or_warning')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.get_status')
@patch('feed_ingestion.tasks.reporter_tasks.garcon_feed_status.set_status')
@patch('feed_ingestion.tasks.reporter_tasks.s3')
@patch('feed_ingestion.tasks.reporter_tasks.os')
@patch('feed_ingestion.tasks.reporter_tasks.itunes_reporter')
@patch('feed_ingestion.tasks.reporter_tasks._get_cred')
@patch('feed_ingestion.util.task_status.add_newcontext')
@patch('feed_ingestion.util.task_status.update_report_context_status')
@patch('feed_ingestion.util.task_status.get_report_contexts')
def test_file_rename_different_versions_in_reporter_file_to_s3(
mock_get_report_contexts,
mock_update_context_status,
mock_add_new_context,
mock__get_cred,
mock_itunes_reporter, mock_os, mock_s3,
mock_set_feed_status, mock_get_feed_status,
mock_send_error_or_warning):
"""Test extract_reporter_file_to_s3."""
# setup mocks
mock_reporter = MagicMock()
file_name = 'file01_V1_0.txt.gz'
reporter_file_name = 'file01_V1_1.txt.gz'
reporter_return = \
'' \
'' \
.format(reporter_file_name)
mock_reporter.get_sales_report_file_name.return_value = file_name
mock_reporter.download_sales_report.return_value = reporter_return
mock_itunes_reporter.get_reporter.return_value = mock_reporter
reporter_account = 'ORCHARD'
report_type = 'amStreams'
date = '2016-01-01'
destination_s3_path = 's3://foo/bar/'
feed_name = 'Feed'
# test successful file download with feed_status calls
reporter_tasks.extract_reporter_file_to_s3(
activity.Activity(boto3.client('swf', 'us-east-1')),
reporter_account, report_type, date,
destination_s3_path, feed_name)
# assert reporter calls
# ensure the file was not renamed by _handle_file_name_incorrection
assert mock_os.rename.call_count == 0
assert mock_s3.upload_to_s3.call_args_list == [
call(
file_path='file01_V1_0.txt.gz',
bucket_name='foo',
object_key='bar/file01_V1_0.txt.gz',
)]
@patch.object(reporter_tasks, 'SWFSecretsManager')
@pytest.mark.parametrize(
'cred_name, is_optional, expected', [
('existing_cred', True, 'value'),
('existing_cred', False, 'value'),
('not_existing_cred', True, None),
('not_existing_cred', False, reporter_tasks.ClientError),
]
)
def test__get_cred(mock_secrets_manager, cred_name, is_optional, expected):
"""Test _get_cred."""
def get_cred_side_effect(name):
if name == 'existing_cred':
return 'value'
raise reporter_tasks.ClientError(
error_response={'Error': {'Code': 'ResourceNotFoundException'}},
operation_name='Some'
)
mock_secrets_manager.return_value.get_cred.side_effect = (
get_cred_side_effect)
try:
result = reporter_tasks._get_cred(
cred_name=cred_name,
optional=is_optional)
assert result == expected
except Exception as e:
assert isinstance(e, expected)
assert mock_secrets_manager.call_args_list == [
call(environment='dev', service_name='itunesconnect')
]
@patch.object(reporter_tasks, '_get_cred')
@pytest.mark.parametrize(
'licensor, expected_value', [
('existing_licensor', 'value_licensor'),
('not_existing_licensor', 'value_common'),
]
)
def test__get_access_token(mock_get_cred, licensor, expected_value):
"""Test _get_access_token."""
def get_cred_side_effect(cred_name, optional):
creds = {
'AccessToken_existing_licensor': 'value_licensor',
'AccessToken': 'value_common',
}
return creds.get(cred_name)
mock_get_cred.side_effect = get_cred_side_effect
result = reporter_tasks._get_access_token(
licensor=licensor)
assert result == expected_value
@patch('feed_ingestion.tasks.reporter_tasks.os')
@pytest.mark.parametrize(
'reporter_response_filename, expected_filename, file_is_renamed', [
('AppleMusic_SongUniques_80028967_20230801_V1_0_[1-2].txt.gz',
'AppleMusic_SongUniques_80028967_20230801_V1_0.txt.gz', True),
('AppleMusic_SongUniques_80028967_20230801_V1_1.zip',
'AppleMusic_SongUniques_80028967_20230801_V1_0.txt.gz', False),
('AppleMusic_LibraryEvents_80012345_20210101.txt.gz',
'AppleMusic_LibraryEvents_80012345_20210101_V5_0.txt.gz', False),
('AppleMusic_LibraryEvents_80012345_20210101_[5-2].txt.gz',
'AppleMusic_LibraryEvents_80012345_20210101_V5_0.txt.gz', False),
('AppleMusic_LibraryEvents_80012345_20210101_V5_0_[5-2].txt.gz',
'AppleMusic_LibraryEvents_80012345_20210101_V5_0.txt.gz', True),
('iTunes_ContentDemographics_80012345_20170320_V1_2_[1-2].zip',
'iTunes_ContentDemographics_80012345_20170320_V1_2.zip', True),
('AppleMusic_SummaryStreams_80012345_20210101_V1_1.zip',
' AppleMusic_SummaryStreams_80012345_20210101_V1_2.zip', False),
# incorrect behaviour
('AppleMusic_SongUniques_80028967_20241014_V1_0_[1-2].txt.gz',
'AppleMusic_SongUniques_80028967_20241014.txt.gz', True),
]
)
def test__handle_file_name_incorrection(mock_os, reporter_response_filename,
expected_filename,
file_is_renamed):
"""Test _handle_file_name_incorrection."""
reporter_resp = f"""
"""
mock_activity = MagicMock()
_handle_file_name_incorrection(reporter_resp,
expected_filename, mock_activity)
if file_is_renamed:
mock_os.rename.assert_called_with(reporter_response_filename,
expected_filename)
else:
assert mock_os.rename.call_count == 0