"""Unit tests for tasks of iTunes Marketshare Workflow.""" from unittest.mock import MagicMock from unittest.mock import patch from boto3.exceptions import S3UploadFailedError from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.flows.itunes_marketshare import config from feed_ingestion.flows.itunes_marketshare import tasks _date = '2018-05-01' @pytest.fixture def mock_set_overall_status(): """Yield overall status.""" overall_status_path = ( 'feed_ingestion.flows.itunes_marketshare.tasks.' 'garcon_feed_status.set_overall_status') with patch(overall_status_path) as overall_status: yield overall_status @pytest.fixture() def input_drop_files(): """Mock drop files.""" return [ ( 'S2_80027963_0918_AU.txt', 'Apple Music Cover Sheet\n' 'Fiscal Month September 2018\n' 'Period 08/31/2018-09/27/2018\n' 'Currency AUD\n\n' ' Australia\n' 'Total Royalty Amount 47.98\n' 'Average Effective Per Play Rate .021128\n\n' 'LINEAR RADIO SERVICE\t\n' 'Total Royalty Bearing Broadcast Plays -\n' 'Label Royalty Bearing Broadcast Plays -\n' 'Label Market Share -\n\t\n' 'Total Listener Hours -\n' 'Revenue per Thousand Listener Hours Guarantee 29.770000000\n' 'Divisor 1000\n' 'Revenue per Thousand Listener Hour -\n' 'RPM Amount -\n\t\n' 'Royalty Amount -\n' 'Effective Per Play Rate -\n\t\n' 'FAMILY MONTHLY PAID\t\n' 'Total Royalty Bearing Plays 129808179.000\n' 'Label Royalty Bearing Plays 778.000\n' 'Label Market Share .0000059934590100058332996\n\t\n' '08/19/2016-11/07/2017\n' 'Revenue -\n' 'Percentage of Revenue 58.000000000\n' 'Revenue Share Royalty Amount -\n\t\n' 'Total Subscription Days -\n' 'Per Subscription Daily Minimum .316000000\n' 'Subscription Minima Royalty Amount -\n\t\n' '11/07/2017-09/27/2018\n' 'Revenue 1886698.21\n' 'Percentage of Revenue 55.000000000\n' 'Revenue Share Royalty Amount 6.22\n\t\n' 'Total Subscription Days 3461886.600000000\n' 'Per Subscription Daily Minimum .300000000\n' 'Subscription Minima Royalty Amount 6.22\n\t\n' 'Royalty Amount 6.22\n' 'Effective Per Play Rate .007995\n\t\n' 'indirect individual monthly paid\t' ) ] @pytest.fixture def expected_processed_file_content(): """Preprocessed file content.""" return ( 'country\tcurrency\tdistributor\tDivisor\t' 'Effective Per Play Rate\tend_date\tfilename\t' 'Label Market Share\tLabel Paid Subscriber Royalty Bearing Plays\t' 'Label Proportionate Share\tLabel Royalty Bearing Broadcast Plays\t' 'Label Royalty Bearing Plays\tPaid Subscriber Market Share\t' 'Per Play Amount\tPer Play Rate\tPer Subscriber Daily Minimum\t' 'Per Subscriber Minimum Amount\tPer Subscription Daily Minimum\t' 'Percentage of Revenue\tproduct\tRPM Amount\tRevenue\t' 'Revenue Share Amount\tRevenue Share Royalty Amount\t' 'Revenue per Thousand Listener Hour\t' 'Revenue per Thousand Listener Hours Guarantee\t' 'Royalty Amount\tstart_date\tSubscription Minima Royalty Amount\t' 'Total Listener Hours\tTotal Paid Subscriber Royalty Bearing Plays\t' 'Total Royalty Bearing Broadcast Plays\tTotal Royalty Bearing Plays\t' 'Total Subscriber Days\tTrial Royalty Pool\t' 'Trial Share Royalty Amount\tis_trial\tis_adsup\t' 'is_multiuser\tis_bundle\t' 'Eligible Account Discount Rate\tRoyalty Subtotal\r\n' 'Australia\tAUD\t80027963\t1000.0\t\t09/27/2018\t' 'S2_80027963_0918_AU.txt\t\t\t\t\t\t\t\t\t\t\t\t\t' 'LINEAR RADIO\t\t\t\t\t\t29.77\t\t08/31/2018' '\t\t\t\t\t\t\t\t\tFalse\tTrue\tFalse\tFalse\t\t\r\n' 'Australia\tAUD\t80027963\t\t0.007995\t09/27/2018\t' 'S2_80027963_0918_AU.txt\t5.993459010005833e-06\t\t\t\t' '778.0\t\t\t\t\t\t0.3\t55.0\tFAMILY MONTHLY PAID\t\t' '1886698.21\t\t6.22\t\t\t6.22\t08/31/2018\t6.22\t\t\t\t' '129808179.0\t3461886.6\t\t\tFalse\tFalse\tTrue\tFalse\t\t\r\n' ) @pytest.fixture def process_drop_files_context(expected_bootstrap_response): """Context for task process_drop_files.""" source_files_dict = { 'files': [ { 'file_name': 'S2_80027963_0918_US.txt', 'found': True, 'file_size': 1}, { 'file_name': 'S2_80027963_0918_AU.txt', 'found': True, 'file_size': 1}]} return { 'activity': MagicMock(), 'feed_name': config.feed_name, 'date': _date, 's3_archive_path': expected_bootstrap_response['s3_archive_path'], 's3_preprocessed_path': expected_bootstrap_response[ 's3_preprocessed_path'], 'preprocessed_filename': expected_bootstrap_response[ 'preprocessed_filename'], 'source_files_dict': source_files_dict} @pytest.fixture def expected_bootstrap_response(): """Response for bootstrap task.""" return { 'feed_name': config.feed_name, 'secrets_path': config.secrets_path, 'date': _date, 'staging_raw_table': 'staging_raw_itunes_market_share', 'file_pattern': '.+S2.+', 'preprocessed_filename': 'itunes_mkt_share_2018-05-01.tsv.gz', 's3_archive_path': 's3://dev-cucumbers/iTunesMarketshare/archives/2018-05/', 's3_preprocessed_path': 's3://dev-cucumbers/iTunesMarketshare/preprocessed/', 'itunes_reports': ( ('ORCHARD', 'AppleMusicSummary'), ('IODA', 'AppleMusicSummary'), ('BFM_ORCHARD', 'AppleMusicSummary'), ('BFM_IODA', 'AppleMusicSummary'), ('FINETUNES', 'AppleMusicSummary'), ('RELAPSE', 'AppleMusicSummary'), ('PRIDDIS', 'AppleMusicSummary'), ('RDA', 'AppleMusicSummary'), ('IRIS', 'AppleMusicSummary'), ('PHONOFILE', 'AppleMusicSummary'), ('DKUA', 'AppleMusicSummary'), ('AWAL', 'AppleMusicSummary'), ('ARTSPAGES', 'AppleMusicSummary') ) } def test_bootstrap(expected_bootstrap_response): """Test bootstrap task.""" result = tasks.bootstrap( activity=MagicMock(), date=_date, reload=None) assert result == expected_bootstrap_response @patch('feed_ingestion.flows.itunes_marketshare.tasks._process_data') @patch('feed_ingestion.flows.itunes_marketshare.tasks.s3utils') def test_process_drop_files_with_upload_error( mock_s3utils, mock_process_data, process_drop_files_context, mock_set_overall_status): """Should set status NOT_AVAILABLE.""" def sf_raise_error(csvfile, s3_path, expected_bucket_owner=config.expected_bucket_owner): raise S3UploadFailedError(500, 'Service error.') mock_s3utils.upload_processed_to_s3.side_effect = sf_raise_error with pytest.raises(S3UploadFailedError): tasks.process_drop_files(**process_drop_files_context) assert mock_process_data.called assert mock_s3utils.upload_processed_to_s3.called mock_set_overall_status.assert_called_with( '_'.join([config.feed_name, config.feed_name]), _date, garcon_feed_status.STATUS_NOT_AVAILABLE) @patch('feed_ingestion.flows.itunes_marketshare.tasks.s3utils') def test_process_drop_files( mock_s3utils, process_drop_files_context, input_drop_files, expected_processed_file_content): """Should process and upload files.""" filename = process_drop_files_context['preprocessed_filename'] expected_response = { 'source_files_dict': { 'files': [ {'file_name': filename, 'found': True, 'file_size': 1}]}} mock_s3utils.upload_processed_to_s3.return_value = expected_response mock_s3utils.get_source_files_content.return_value = input_drop_files result = tasks.process_drop_files(**process_drop_files_context) preprocessed_content = (mock_s3utils.upload_processed_to_s3 .call_args[0][0].getvalue()) assert preprocessed_content == expected_processed_file_content assert mock_s3utils.upload_processed_to_s3.called assert result == expected_response @patch.object(tasks, 'itunes_reporter') def test_get_vendors_and_regions(mock_itunes_reporter): """Test get_vendors_and_regions task.""" reporter = mock_itunes_reporter.get_reporter() reporter.get_available_regions = MagicMock(return_value=['AE', 'BG']) result = tasks.get_vendors_and_regions( activity=MagicMock(), vendor='ORCHARD', report_type='AppleMusicSummary') assert 'AE' and 'BG' in result['ORCHARD'] reporter.get_available_regions = MagicMock(return_value=['UA', 'UK']) result = tasks.get_vendors_and_regions( activity=MagicMock(), vendor='ORCHARD', report_type='AppleMusicSummary') assert 'UA' and 'UK' in result['ORCHARD'] reporter.get_available_regions = MagicMock( return_value=['BG', 'AE', 'US', 'TR']) result = tasks.get_vendors_and_regions( activity=MagicMock(), vendor='ORCHARD', report_type='AppleMusicSummary') assert 'AE' and 'US' and 'BG' and 'TR' in result['ORCHARD'] assert mock_itunes_reporter.get_reporter.assert_called assert mock_itunes_reporter.get_available_regions.assert_called