"""Unit tests for AWA tasks workflow.""" from pathlib import Path from unittest.mock import MagicMock, call from unittest.mock import patch from botocore.exceptions import ClientError from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.flows.awa import tasks _date = '2021-08-31' @pytest.fixture def feed_name(): """Feed name value.""" return 'awa' @pytest.fixture def source_file_name(): """Source files value.""" return '20210831_play_summary_397.tsv' @pytest.fixture def source_bucket_name(): """Source files value.""" return 'test_bucket' @patch.object(tasks, 'garcon_feed_status') def test_bootstrap_theorchard(garcon_feed_status_mock): """Test bootstrap.""" garcon_feed_status_mock.get_overall_status.return_value = 'NOT_INGESTED' result = tasks.bootstrap( MagicMock(), _date, reload=False, licensor='theorchard') expected_bootstrap_response = { 'feed_name': 'awa_theorchard', 'date': '2021-08-31', 'drop_path': 'feed-drop/AWA/', 'archive_path': 'AWA/archives/2021-08-31/', 'source_file_name': '20210831_play_summary_397.tsv', 'source_bucket_name': 'dev-feed-drop', 'clean_path': 'AWA/archives/2021-08-31/clean/', 'secrets_path': 'awa', 'stop_after_staging_raw': False, 'licensor': 'theorchard', 'fact_table_report': 'play_summary', 'staging_raw_table': 'staging_raw_awa', } assert result == expected_bootstrap_response @patch.object(tasks, 'garcon_feed_status') def test_bootstrap_smej(garcon_feed_status_mock): """Test bootstrap.""" garcon_feed_status_mock.get_overall_status.return_value = 'NOT_INGESTED' result = tasks.bootstrap( MagicMock(), _date, reload=False, licensor='smej') expected_bootstrap_response = { 'feed_name': 'awa_smej', 'date': '2021-08-31', 'drop_path': 'feed-drop/AWA/sme/', 'archive_path': 'AWA/smej/archives/2021-08-31/', 'source_file_name': 'PKZ0_D_20210831_20210831.txt', 'source_bucket_name': 'dev-feed-drop', 'clean_path': 'AWA/smej/archives/2021-08-31/clean/', 'secrets_path': 'awa', 'stop_after_staging_raw': True, 'licensor': 'smej', } assert result == expected_bootstrap_response @pytest.fixture def mock_task_status(): """Yield task status.""" task_status_path = 'feed_ingestion.tasks.task_status' with patch(task_status_path) as task_status: task_status.is_completed_task.return_value = False task_status.mark_completed_task = MagicMock() yield task_status @pytest.fixture def mock_set_overall_status(): """Yield overall status.""" overall_status_path = ( 'feed_ingestion.flows.awa.tasks.garcon_feed_status.' 'set_overall_status') with patch(overall_status_path) as overall_status: yield overall_status @pytest.fixture def mock_set_missing_files(): """Yield set_missing_files.""" path = ( 'feed_ingestion.flows.awa.tasks.garcon_feed_status.' 'set_missing_files') with patch(path) as mock_set_missing: yield mock_set_missing @pytest.fixture def mock_s3_tasks(): """Mock s3_tasks.""" path = 'feed_ingestion.flows.awa.tasks.s3_tasks' with patch(path) as mock_s3: yield mock_s3 def test_grab_drop_files( mock_s3_tasks, mock_set_overall_status, mock_task_status, mock_set_missing_files, source_file_name, source_bucket_name): """Test grab_drop_files.""" mock_s3_tasks.copy_file.return_value = \ {'20210831_play_summary_397.tsv': True} activity = MagicMock() result = tasks.grab_drop_files( activity=activity, feed_name='awa', date=_date, source_bucket_name=source_bucket_name, source_file_name=source_file_name, drop_path='drop_path', archive_path='archive_path') mock_set_overall_status.assert_called_once_with( 'awa', _date, garcon_feed_status.STATUS_DOWNLOADED) mock_set_missing_files.assert_not_called() assert result == {'20210831_play_summary_397.tsv': True} @pytest.fixture def mock_tempdir(): with patch.object(tasks, 'tempfile') as mock_tempfile: temp_dir_mock = mock_tempfile.TemporaryDirectory.return_value.__enter__ temp_dir_mock.return_value = '/tmpdirmock' yield temp_dir_mock @patch.object(tasks, 'boto3') @patch.object(tasks, 'convert_reporting_txt_to_csv') def test_grab_drop_files_smej( mock_convert, mock_boto3, mock_tempdir, mock_set_overall_status, mock_task_status, mock_set_missing_files): """Test grab_drop_files_smej.""" activity = MagicMock() s3_client_mock = mock_boto3.client.return_value result = tasks.grab_drop_files_smej( activity=activity, feed_name='awa_smej', date=_date, source_bucket_name='dev-feed-drop', source_file_name='PKZ0_D_20210831_20210831.txt', drop_path='drop_path/', clean_path='clean_path/', archive_path='archive_path/') assert s3_client_mock.download_file.call_args_list == [ call('dev-feed-drop', 'drop_path/PKZ0_D_20210831_20210831.txt', '/tmpdirmock/PKZ0_D_20210831_20210831.txt') ] assert s3_client_mock.upload_file.call_args_list == [ call('/tmpdirmock/PKZ0_D_20210831_20210831.txt', 'dev-cucumbers', 'archive_path/PKZ0_D_20210831_20210831.txt'), call('/tmpdirmock/standard_sales.tsv', 'dev-cucumbers', 'clean_path/standard_sales.tsv'), call('/tmpdirmock/marketshare.tsv', 'dev-cucumbers', 'clean_path/marketshare.tsv'), ] assert mock_convert.call_args_list == [ call(source_path='/tmpdirmock/PKZ0_D_20210831_20210831.txt', target_path='/tmpdirmock/standard_sales.tsv', filter_by_record_type='N'), call(source_path='/tmpdirmock/PKZ0_D_20210831_20210831.txt', target_path='/tmpdirmock/marketshare.tsv', filter_by_record_type='M') ] mock_set_overall_status.assert_called_once_with( 'awa_smej', _date, 'DOWNLOADED') assert mock_set_missing_files.call_args_list == [ ] assert result == {} @patch.object(tasks, 'boto3') @patch.object(tasks, 'convert_reporting_txt_to_csv') def test_grab_drop_files_smej_no_file( mock_convert, mock_boto3, mock_tempdir, mock_set_overall_status, mock_task_status, mock_set_missing_files): """Test grab_drop_files_smej.""" activity = MagicMock() s3_client_mock = mock_boto3.client.return_value s3_client_mock.download_file.side_effect = ClientError( error_response={ 'Error': { 'Code': '404', 'Message': 'Not Found' } }, operation_name='download_file', ) result = tasks.grab_drop_files_smej( activity=activity, feed_name='awa_smej', date=_date, source_bucket_name='dev-feed-drop', source_file_name='PKZ0_D_20210831_20210831.txt', drop_path='drop_path/', clean_path='clean_path/', archive_path='archive_path/') assert s3_client_mock.download_file.call_args_list == [ call('dev-feed-drop', 'drop_path/PKZ0_D_20210831_20210831.txt', '/tmpdirmock/PKZ0_D_20210831_20210831.txt') ] assert s3_client_mock.upload_file.call_args_list == [] assert mock_convert.call_args_list == [] mock_set_overall_status.assert_called_once_with( 'awa_smej', _date, 'NOT_AVAILABLE') assert mock_set_missing_files.call_args_list == [ call('awa_smej', _date, ['PKZ0_D_20210831_20210831.txt']) ] assert result == {'stop': True} @patch.object(tasks, 'boto3') @patch.object(tasks, 'convert_reporting_txt_to_csv') def test_grab_drop_files_smej_connection_error( mock_convert, mock_boto3, mock_tempdir, mock_set_overall_status, mock_task_status, mock_set_missing_files): """Test grab_drop_files_smej.""" activity = MagicMock() s3_client_mock = mock_boto3.client.return_value s3_client_mock.download_file.side_effect = ClientError( error_response={ 'Error': { 'Code': '500', 'Message': 'Other error' } }, operation_name='download_file', ) with pytest.raises(ClientError): tasks.grab_drop_files_smej( activity=activity, feed_name='awa_smej', date=_date, source_bucket_name='dev-feed-drop', source_file_name='PKZ0_D_20210831_20210831.txt', drop_path='drop_path/', clean_path='clean_path/', archive_path='archive_path/') assert s3_client_mock.download_file.call_args_list == [ call('dev-feed-drop', 'drop_path/PKZ0_D_20210831_20210831.txt', '/tmpdirmock/PKZ0_D_20210831_20210831.txt') ] assert s3_client_mock.upload_file.call_args_list == [] assert mock_convert.call_args_list == [] mock_set_overall_status.assert_called_once_with( 'awa_smej', _date, 'NOT_AVAILABLE') assert mock_set_missing_files.call_args_list == [ call('awa_smej', _date, ['PKZ0_D_20210831_20210831.txt']) ] @pytest.mark.parametrize( 'record_type, expected_file', [ ('N', 'standard_sales.tsv'), ('M', 'marketshare.tsv') ] ) def test_convert_reporting_txt_to_csv(tmp_path, record_type, expected_file): DATA_DIR = Path(__file__).parent / 'data' input_file = DATA_DIR / 'PKZ0_D_20240707_20240707.txt' # Create a temporary output file output_file = tmp_path / 'output.csv' # Call the function under test tasks.convert_reporting_txt_to_csv( source_path=str(input_file), target_path=str(output_file), filter_by_record_type=record_type) expected_file = DATA_DIR / expected_file assert output_file.read_text() == expected_file.read_text() def test_grab_drop_files_no_file( mock_s3_tasks, mock_set_overall_status, mock_task_status, mock_set_missing_files): """Test grab_drop_files if there is no file.""" mock_s3_tasks.copy_file.return_value = \ {'20210831_play_summary_397.tsv': False} activity = MagicMock() result = tasks.grab_drop_files( activity=activity, feed_name='awa', date=_date, source_bucket_name='dev-feed-drop', source_file_name='20210831_play_summary_397.tsv', drop_path='drop_path', archive_path='archive_path') mock_set_overall_status.assert_called_once_with( 'awa', _date, garcon_feed_status.STATUS_NOT_AVAILABLE) mock_set_missing_files.assert_called_once_with( 'awa', _date, ['20210831_play_summary_397.tsv']) assert result == {'stop': True}