"""Unit tests for WYNK tasks workflow.""" from unittest.mock import MagicMock from unittest.mock import patch from botocore.exceptions import ClientError from garcon_contrib.dynamo_feed_status import garcon_feed_status import pytest from feed_ingestion.flows.wynk import tasks _date = '2022-06-10' @pytest.fixture def feed_name(): """Feed name value.""" return 'wynk' @pytest.fixture def source_file_name(): """Source files value.""" return 'orchard-daily-cms.csv' @pytest.fixture def expected_bootstrap_response(): """Response for bootstrap task.""" return { 'feed_name': 'wynk', 'date': '2022-06-10', 'drop_path': 'feed-drop/wynk/2022-06-10/', 'archive_path': 'wynk/archives/2022-06-10/', 'source_file_name': 'orchard-daily-cms.csv', 's3_archive_path': 's3://dev-cucumbers/wynk/archives/2022-06-10/', 'key_dir': 's3://dev-cucumbers/wynk/archives/2022-06-10/' 'orchard-daily-cms.csv', 'secrets_path': 'swf-wynk', 'staging_raw_table': 'staging_raw_wynk', 'temp_staging_raw_table': 'temp_staging_raw_wynk_2022_06_10' } def test_bootstrap(expected_bootstrap_response): """Test bootstrap.""" result = tasks.bootstrap(MagicMock(), _date, reload=False) assert result == expected_bootstrap_response @pytest.fixture def mock_task_status(): """Yield task status.""" task_status_path = 'feed_ingestion.tasks.task_status' with patch(task_status_path) as task_status: task_status.is_completed_task.return_value = False task_status.mark_completed_task = MagicMock() yield task_status @pytest.fixture def mock_set_overall_status(): """Yield overall status.""" overall_status_path = ( 'feed_ingestion.flows.wynk.tasks.garcon_feed_status.' 'set_overall_status') with patch(overall_status_path) as overall_status: yield overall_status @pytest.fixture def mock_set_missing_files(): """Yield set_missing_files.""" path = ( 'feed_ingestion.flows.wynk.tasks.garcon_feed_status.' 'set_missing_files') with patch(path) as mock_set_missing: yield mock_set_missing @pytest.fixture def mock_s3_tasks(): """Mock s3_tasks.""" path = 'feed_ingestion.flows.wynk.tasks.s3_tasks' with patch(path) as mock_s3: yield mock_s3 @pytest.fixture def mock_delete_status(): """Yield delete overall status.""" path = 'feed_ingestion.flows.wynk.tasks.garcon_feed_status.delete_status' with patch(path) as delete_status: yield delete_status @pytest.fixture def mock_boto3(): """Mock boto3.""" boto3_path = 'feed_ingestion.flows.wynk.tasks.boto3' with patch(boto3_path) as boto3: mock_client = MagicMock() boto3.client.return_value = mock_client yield mock_client @pytest.fixture def mock_shutil(): """Mock shutil.""" path = 'feed_ingestion.flows.wynk.tasks.shutil' with patch(path) as shutil: yield shutil @pytest.fixture def mock_os(): """Mock os.""" path = 'feed_ingestion.flows.wynk.tasks.os.makedirs' with patch(path) as mock_makedirs: yield mock_makedirs def test_grab_drop_files( mock_s3_tasks, mock_set_overall_status, mock_task_status, mock_set_missing_files, source_file_name): """Test grab_drop_files.""" mock_s3_tasks.copy_file.return_value = \ {'orchard-daily-cms.csv': True} activity = MagicMock() result = tasks.grab_drop_files( activity=activity, feed_name='wynk', date=_date, source_file_name=source_file_name, drop_path='drop_path', archive_path='archive_path') mock_set_overall_status.assert_called_once_with( 'wynk', _date, garcon_feed_status.STATUS_DOWNLOADED) mock_set_missing_files.assert_not_called() assert result == \ {'orchard-daily-cms.csv': True} def test_grab_drop_files_no_file( mock_s3_tasks, mock_set_overall_status, mock_task_status, mock_set_missing_files): """Test grab_drop_files if there is no file.""" mock_s3_tasks.copy_file.return_value = \ {'orchard-daily-cms.csv': False} activity = MagicMock() result = tasks.grab_drop_files( activity=activity, feed_name='wynk', date=_date, source_file_name='orchard-daily-cms.csv', drop_path='drop_path', archive_path='archive_path') mock_set_overall_status.assert_called_once_with( 'wynk', _date, garcon_feed_status.STATUS_NOT_AVAILABLE) mock_set_missing_files.assert_called_once_with( 'wynk', _date, ['orchard-daily-cms.csv']) assert result == {'stop': True} def test_upload_file( mock_task_status, mock_boto3, mock_set_overall_status, mock_delete_status, mock_shutil, mock_os): """Test upload_file.""" tasks.upload_file( activity=MagicMock(), feed_name='wynk', date=_date, archive_path='archive_path/', source_file_name='source_file_name' ) mock_boto3.download_file.assert_called_once_with( 'dev-cucumbers', 'archive_path/source_file_name', './file_stage_2022-06-10/source_file_name') assert mock_boto3.upload_file.call_count == 1 mock_delete_status.assert_called_with('wynk', '2022-06-10') mock_set_overall_status.assert_called_with( 'wynk', '2022-06-10', 'NOT_INGESTED') mock_shutil.rmtree.assert_called_once_with('./file_stage_2022-06-10') def test_upload_file_failed_while_file_upload( mock_task_status, mock_boto3, mock_set_overall_status, mock_delete_status, mock_shutil, mock_os): """Test upload_file if copy files on s3 failed.""" mock_boto3.upload_file.side_effect = ClientError( {'Error': {'Code': '404', 'Message': 'Not found'}}, 'Not found') with pytest.raises(ClientError): tasks.upload_file( activity=MagicMock(), feed_name='wynk', date=_date, archive_path='archive_path/', source_file_name='source_file_name' ) mock_boto3.download_file.assert_called_once_with( 'dev-cucumbers', 'archive_path/source_file_name', './file_stage_2022-06-10/source_file_name') assert mock_boto3.upload_file.call_count == 1 mock_delete_status.assert_called_once_with('wynk', _date) mock_set_overall_status.assert_called_once_with( 'wynk', _date, garcon_feed_status.STATUS_NOT_INGESTED) mock_shutil.rmtree.assert_called_once_with('./file_stage_2022-06-10')