"""Unit tests for GfK Physical tasks.""" import io from unittest.mock import MagicMock, patch import zipfile import pytest from feed_ingestion.flows.gfk_physical import tasks _date = '2026-04-26' _filename = 'BM260426CSV.ZIP' _archive_path = 'GfK_Physical/archives/2026-04-26/BM260426CSV.ZIP' _processed_path = 'GfK_Physical/processed/2026-04-26/BM260426CSV.csv' @pytest.fixture def mock_get_overall_status(): """Yield get_overall_status mock.""" with patch( 'feed_ingestion.flows.gfk_physical.tasks.' 'garcon_feed_status.get_overall_status' ) as mock_obj: yield mock_obj @pytest.fixture def mock_set_overall_status(): """Yield set_overall_status mock.""" with patch( 'feed_ingestion.flows.gfk_physical.tasks.' 'garcon_feed_status.set_overall_status' ) as mock_obj: yield mock_obj @pytest.fixture def mock_delete_status(): """Yield delete_status mock.""" with patch( 'feed_ingestion.flows.gfk_physical.tasks.' 'garcon_feed_status.delete_status' ) as mock_obj: yield mock_obj @pytest.fixture def expected_bootstrap_response(): """Return expected response from bootstrap task.""" return { 'feed_name': 'gfk_physical', 'date': '2026-04-26', 'filename': 'BM260426CSV.ZIP', 'archive_path': 'GfK_Physical/archives/2026-04-26/BM260426CSV.ZIP', 'processed_path': 'GfK_Physical/processed/2026-04-26/BM260426CSV.csv', 'staging_raw_table': 'staging_raw_gfk_physical', 's3_dir_path': 's3://dev-cucumbers/GfK_Physical/processed/2026-04-26/', } class TestBootstrap: """Tests for the bootstrap task.""" @pytest.mark.parametrize( 'input_date,expected_date,expected_filename', [ ('2026-04-26', '2026-04-26', 'BM260426CSV.ZIP'), ('2026-04-27', '2026-04-27', 'BM260427CSV.ZIP'), ('2026-04-28', '2026-04-28', 'BM260428CSV.ZIP'), ], ) def test_bootstrap_file_pattern( self, mock_get_overall_status, mock_delete_status, input_date, expected_date, expected_filename, ): """Test that bootstrap produces correct file names for given dates.""" result = tasks.bootstrap(MagicMock(), input_date, reload=None) assert result['date'] == expected_date assert result['filename'] == expected_filename def test_bootstrap_normal( self, expected_bootstrap_response, mock_get_overall_status, mock_delete_status, ): """Test bootstrap with normal execution.""" result = tasks.bootstrap(MagicMock(), _date, reload=None) assert result == expected_bootstrap_response mock_delete_status.assert_not_called() def test_bootstrap_reload( self, expected_bootstrap_response, mock_get_overall_status, mock_delete_status, ): """Test bootstrap with reload=True deletes the DynamoDB status.""" result = tasks.bootstrap(MagicMock(), _date, reload='True') assert result == expected_bootstrap_response mock_delete_status.assert_called_once_with( 'gfk_physical', '2026-04-26' ) def test_bootstrap_already_ingested(self, mock_get_overall_status): """Test bootstrap returns stop when workflow is already ingested.""" mock_get_overall_status.return_value = 'INGESTED' result = tasks.bootstrap(MagicMock(), _date, reload=None) assert result == { 'stop': True, 'message': 'gfk_physical is already ingested for 2026-04-26', } @pytest.fixture def mock_task_status(monkeypatch): """Bypass DynamoDB in check_status decorator.""" from feed_ingestion.util import task_status monkeypatch.setattr( task_status, 'is_completed_task', MagicMock(return_value=False) ) monkeypatch.setattr(task_status, 'mark_completed_task', MagicMock()) @pytest.fixture def mock_sftp_connection(): """Yield a mocked SFTP client.""" with patch( 'feed_ingestion.flows.gfk_physical.tasks._get_sftp_connection' ) as mock_fn: mock_client = MagicMock() mock_fn.return_value = mock_client yield mock_client @pytest.fixture def mock_s3_client(): """Yield a mocked boto3 S3 client.""" with patch( 'feed_ingestion.flows.gfk_physical.tasks.boto3.client' ) as mock_boto3: mock_client = MagicMock() mock_boto3.return_value = mock_client yield mock_client def _make_zip_with_csv(csv_content=b'col1,col2\nval1,val2\n'): """Build an in-memory ZIP containing a single CSV file.""" buf = io.BytesIO() with zipfile.ZipFile(buf, 'w') as zf: zf.writestr('data.csv', csv_content) buf.seek(0) return buf.read() @pytest.fixture def mock_zip_on_disk(tmp_path, monkeypatch): """Write a valid ZIP to the temp path that sftp.get would produce.""" zip_bytes = _make_zip_with_csv() def fake_get(remote_path, local_path): with open(local_path, 'wb') as f: f.write(zip_bytes) return fake_get class TestFetchFromSftp: """Tests for the fetch_from_sftp task.""" def test_file_not_found_returns_stop( self, mock_task_status, mock_sftp_connection, mock_set_overall_status, ): """Test that a missing SFTP file returns a stop dict.""" mock_sftp_connection.stat.side_effect = FileNotFoundError result = tasks.fetch_from_sftp( MagicMock(), 'gfk_physical', _date, _filename, _archive_path, _processed_path, ) assert result == {'stop': True, 'message': f'Missing file {_filename}'} mock_set_overall_status.assert_called_once_with( 'gfk_physical', _date, 'NOT_AVAILABLE' ) def test_file_not_found_closes_sftp( self, mock_task_status, mock_sftp_connection, mock_set_overall_status, ): """Test that SFTP connection is closed when file is not found.""" mock_sftp_connection.stat.side_effect = FileNotFoundError tasks.fetch_from_sftp( MagicMock(), 'gfk_physical', _date, _filename, _archive_path, _processed_path, ) mock_sftp_connection.close.assert_called_once() def test_successful_download_uploads_zip_to_archive( self, mock_task_status, mock_sftp_connection, mock_s3_client, mock_set_overall_status, mock_zip_on_disk, monkeypatch, ): """Test that the raw ZIP is uploaded to the S3 archive path.""" mock_sftp_connection.get.side_effect = mock_zip_on_disk monkeypatch.setattr( 'feed_ingestion.flows.gfk_physical.tasks.os.remove', MagicMock() ) tasks.fetch_from_sftp( MagicMock(), 'gfk_physical', _date, _filename, _archive_path, _processed_path, ) mock_s3_client.upload_file.assert_called_once() call_args = mock_s3_client.upload_file.call_args[0] assert call_args[1] == 'dev-cucumbers' assert call_args[2] == _archive_path def test_successful_download_uploads_csv_to_processed_path( self, mock_task_status, mock_sftp_connection, mock_s3_client, mock_set_overall_status, mock_zip_on_disk, monkeypatch, ): """Test that the extracted CSV is uploaded to the S3 processed path.""" mock_sftp_connection.get.side_effect = mock_zip_on_disk monkeypatch.setattr( 'feed_ingestion.flows.gfk_physical.tasks.os.remove', MagicMock() ) tasks.fetch_from_sftp( MagicMock(), 'gfk_physical', _date, _filename, _archive_path, _processed_path, ) mock_s3_client.put_object.assert_called_once() call_kwargs = mock_s3_client.put_object.call_args[1] assert call_kwargs['Bucket'] == 'dev-cucumbers' assert call_kwargs['Key'] == _processed_path assert call_kwargs['Body'] == b'col1,col2\nval1,val2\n' def test_successful_download_sets_downloaded_status( self, mock_task_status, mock_sftp_connection, mock_s3_client, mock_set_overall_status, mock_zip_on_disk, monkeypatch, ): """Test that DOWNLOADED status is set after successful processing.""" mock_sftp_connection.get.side_effect = mock_zip_on_disk monkeypatch.setattr( 'feed_ingestion.flows.gfk_physical.tasks.os.remove', MagicMock() ) tasks.fetch_from_sftp( MagicMock(), 'gfk_physical', _date, _filename, _archive_path, _processed_path, ) mock_set_overall_status.assert_called_once_with( 'gfk_physical', _date, 'DOWNLOADED' ) def test_zip_without_csv_raises( self, mock_task_status, mock_sftp_connection, mock_s3_client, mock_set_overall_status, tmp_path, monkeypatch, ): """Test that a ZIP with no CSV file raises ValueError.""" empty_zip = io.BytesIO() with zipfile.ZipFile(empty_zip, 'w') as zf: zf.writestr('readme.txt', 'no csv here') empty_zip_bytes = empty_zip.getvalue() def fake_get(remote_path, local_path): with open(local_path, 'wb') as f: f.write(empty_zip_bytes) mock_sftp_connection.get.side_effect = fake_get monkeypatch.setattr( 'feed_ingestion.flows.gfk_physical.tasks.os.remove', MagicMock() ) with pytest.raises(ValueError, match='No CSV file found inside ZIP'): tasks.fetch_from_sftp( MagicMock(), 'gfk_physical', _date, _filename, _archive_path, _processed_path, )