"""Tests for feed_file_export.""" import tempfile from types import SimpleNamespace import pytest from unittest.mock import MagicMock import feed_file_exporter as ffe from src.utils.s3_backoff_utils import upload_files from models.snowflake_models import SMEFeedFileHistoryPersister def test_process_regular_feed( mocker, test_period_list, test_group_name_list_message, test_file_name_list, test_get_by_period_group_name, ): """Test process_regular_feed() function.""" table_name = "table_name" tmp_dir = tempfile.TemporaryDirectory() period_count = len(test_period_list) store_count = len(test_group_name_list_message.message) rows_per_file = 3 store_by_period_count = period_count * store_count output_file_list = test_file_name_list(store_by_period_count) mock_get_period_list = mocker.patch( "feed_file_exporter.get_period_list", return_value=test_period_list ) mock_get_group_names_for_period = mocker.patch.object( SMEFeedFileHistoryPersister, "get_group_names_for_period", return_value=test_group_name_list_message, ) mock_get_by_period_group_name = mocker.patch.object( SMEFeedFileHistoryPersister, "get_by_period_group_name", return_value=test_get_by_period_group_name, ) mock_split_rows_to_files = mocker.patch( "feed_file_exporter.split_rows_to_files", return_value=(rows_per_file, ["File.txt"], ["File.txt"]), ) result = ffe.process_regular_feed(table_name, tmp_dir.name) assert mock_get_period_list.call_count == 1 assert mock_get_group_names_for_period.call_count == 2 assert mock_get_by_period_group_name.call_count == 6 assert mock_split_rows_to_files.call_count == 6 assert result[0] == store_by_period_count * rows_per_file assert len(result[1]) == len(output_file_list) def test_process_ex_us_feed( mocker, test_label_list, test_period_list, test_territory_list, test_file_name_list, test_return_rows, test_get_by_period_id, ): """Test process_table_to_files() function with ex-US data.""" table_name = "table_name" tmp_dir = tempfile.TemporaryDirectory() period_count = len(test_period_list) label_count = len(test_label_list) territory_count = len(test_territory_list) rows_per_file = 3 total_count = period_count * label_count * territory_count output_file_list = test_file_name_list(total_count) mock_get_period_list = mocker.patch( "feed_file_exporter.get_period_list", return_value=test_period_list ) mock_get_label_list = mocker.patch( "feed_file_exporter.get_label_list", return_value=test_label_list ) mock_get_territory_list = mocker.patch( "feed_file_exporter.get_territory_list", return_value=test_territory_list ) mock_get_by_period_id = mocker.patch.object( SMEFeedFileHistoryPersister, "get_by_period_id", return_value=test_get_by_period_id, ) mock_split_rows_to_files = mocker.patch( "feed_file_exporter.split_rows_to_files", return_value=(rows_per_file, ["File.txt"], ["File.txt"]), ) result = ffe.process_ex_us_feed(table_name, tmp_dir.name) assert mock_get_period_list.call_count == 1 assert mock_get_label_list.call_count == 1 assert mock_get_territory_list.call_count == 1 assert mock_get_by_period_id.call_count == 8 assert mock_split_rows_to_files.call_count == 8 assert result[0] == total_count * rows_per_file assert len(result[1]) == len(output_file_list) def test_process_aggregate_feed( mocker, test_affiliate_list, test_group_name_list, test_file_name_list, test_return_rows, test_all_rows, test_get_count_by_group_name, test_get_by_booking_affiliate_store, ): """Test process_table_to_files() function with aggregate data.""" table_name = "table_name" tmp_dir = tempfile.TemporaryDirectory() group_count = len(test_group_name_list) # affiliate_count = len(test_affiliate_list) rows_per_file = 3 # total_count = store_count * affiliate_count output_file_list = test_file_name_list(group_count) mock_get_group_name_list = mocker.patch( "feed_file_exporter.get_group_name_list", return_value=test_group_name_list ) mock_get_count_by_group_name = mocker.patch.object( SMEFeedFileHistoryPersister, "get_count_by_group_name", return_value=test_get_count_by_group_name, ) # Mock the streaming context manager mock_stream_ctx = MagicMock() mock_stream_ctx.__enter__.return_value = test_return_rows mock_stream_ctx.__exit__.return_value = None mock_get_by_group_name_stream = mocker.patch.object( SMEFeedFileHistoryPersister, "get_by_group_name_stream", return_value=mock_stream_ctx, ) # Ensure config uses STREAMING mode mocker.patch("config.FETCH_MODE", "STREAMING") mock_split_res_to_files = mocker.patch( "feed_file_exporter.split_res_to_files", return_value=(rows_per_file, ["File.txt"]), ) result = ffe.process_aggregate_feed(table_name, tmp_dir.name) assert mock_get_group_name_list.call_count == 1 assert mock_get_count_by_group_name.call_count == 3 assert mock_get_by_group_name_stream.call_count == 3 assert mock_split_res_to_files.call_count == 3 assert result[0] == group_count * rows_per_file # 27 assert len(result[1]) == len(output_file_list) def test_process_aggregate_feed_chunked( mocker, test_group_name_list, test_file_name_list, test_return_rows, test_get_count_by_group_name, ): """Test process_aggregate_feed with CHUNKED mode.""" table_name = "table_name" tmp_dir = tempfile.TemporaryDirectory() group_count = len(test_group_name_list) rows_per_file = 3 # Based on test_return_rows structure (H, Body, T) mock_get_group_name_list = mocker.patch( "feed_file_exporter.get_group_name_list", return_value=test_group_name_list ) mock_get_count_by_group_name = mocker.patch.object( SMEFeedFileHistoryPersister, "get_count_by_group_name", return_value=test_get_count_by_group_name, ) # Mock chunked fetcher # Use side_effect to return a fresh generator each time mock_get_by_group_name_chunked = mocker.patch.object( SMEFeedFileHistoryPersister, "get_by_group_name_chunked", side_effect=lambda *args, **kwargs: test_return_rows(1), ) # Set config to CHUNKED mocker.patch("config.FETCH_MODE", "CHUNKED") result = ffe.process_aggregate_feed(table_name, tmp_dir.name) assert mock_get_group_name_list.call_count == 1 assert mock_get_count_by_group_name.call_count == 3 assert mock_get_by_group_name_chunked.call_count == 3 # Each group -> 1 chunk -> 3 rows processed assert result[0] == group_count * rows_per_file # Check that files were "created" (returned in list) assert len(result[1]) == group_count def test_process_aggregate_feed_cursor( mocker, test_group_name_list, test_file_name_list, test_return_rows, test_get_count_by_group_name, ): """Test process_aggregate_feed with CURSOR mode.""" table_name = "table_name" tmp_dir = tempfile.TemporaryDirectory() group_count = len(test_group_name_list) rows_per_file = 3 mock_get_group_name_list = mocker.patch( "feed_file_exporter.get_group_name_list", return_value=test_group_name_list ) mock_get_count_by_group_name = mocker.patch.object( SMEFeedFileHistoryPersister, "get_count_by_group_name", return_value=test_get_count_by_group_name, ) # Mock cursor fetcher mock_get_by_group_name_cursor = mocker.patch.object( SMEFeedFileHistoryPersister, "get_by_group_name_cursor", side_effect=lambda *args, **kwargs: test_return_rows(1), ) # Set config to CURSOR mocker.patch("config.FETCH_MODE", "CURSOR") result = ffe.process_aggregate_feed(table_name, tmp_dir.name) assert mock_get_group_name_list.call_count == 1 assert mock_get_count_by_group_name.call_count == 3 assert mock_get_by_group_name_cursor.call_count == 3 assert result[0] == group_count * rows_per_file assert len(result[1]) == group_count def test_process_table_to_files(mocker, test_file_name_list, test_get_all_rows_result): """Test process_table_to_files() function.""" table_name = "table_name" row_count = 6 output_file_list = test_file_name_list(row_count) mock_get_all_rows = mocker.patch.object( SMEFeedFileHistoryPersister, "get_count_of_rows", return_value=test_get_all_rows_result(row_count), ) mock_us_feed = mocker.patch( "feed_file_exporter.process_regular_feed", return_value=(row_count, output_file_list), ) tmp_dir = tempfile.TemporaryDirectory() result = ffe.process_table_to_files(table_name, tmp_dir.name, process="US") assert result == output_file_list assert mock_get_all_rows.call_count == 1 assert mock_us_feed.call_count == 1 def test_process_table_to_files_sap_settlement_uses_aggregate( mocker, test_file_name_list, test_get_all_rows_result, ): """SAP settlement processing reuses the aggregate feed pipeline.""" table_name = 'table_name' row_count = 4 output_file_list = test_file_name_list(row_count) mock_get_all_rows = mocker.patch.object( SMEFeedFileHistoryPersister, 'get_count_of_rows', return_value=test_get_all_rows_result(row_count), ) mock_aggregate_feed = mocker.patch( 'feed_file_exporter.process_aggregate_feed', return_value=(row_count, output_file_list), ) tmp_dir = tempfile.TemporaryDirectory() result = ffe.process_table_to_files( table_name, tmp_dir.name, process='SAP_SETTLEMENT', ) assert result == output_file_list assert mock_get_all_rows.call_count == 1 assert mock_aggregate_feed.call_count == 1 def test_prepare_s3_transfer_files_uses_originals_when_gzip_disabled( monkeypatch, ): """Disabling gzip must still upload the original generated files.""" generated_files = ['tmp/a.TXT', 'tmp/b.TXT'] def _unexpected_gzip(**kwargs): raise AssertionError('gzip helper should not be called') monkeypatch.setattr(ffe, 'gzip_files_in_directory', _unexpected_gzip) result = ffe.prepare_s3_transfer_files( generated_files=generated_files, temp_file_path='tmp/out', gzip_enabled=False, continue_on_error=False, ) assert result == generated_files def test_prepare_s3_transfer_files_returns_gzipped_files(mocker): """Enabling gzip uploads the generated .gz files.""" gzip_result = SimpleNamespace(output_path='tmp/a.TXT.gz') mock_gzip = mocker.patch( 'feed_file_exporter.gzip_files_in_directory', return_value=([gzip_result], []), ) result = ffe.prepare_s3_transfer_files( generated_files=['tmp/a.TXT'], temp_file_path='tmp/out', gzip_enabled=True, continue_on_error=False, ) assert result == ['tmp/a.TXT.gz'] mock_gzip.assert_called_once_with( directory='tmp/out', pattern='*.[Tt][Xx][Tt]', delete_originals=False, compression_level=9, ) def test_prepare_s3_transfer_files_exits_on_gzip_failure(mocker): """Fatal gzip failures abort S3 transfer when continue_on_error=False.""" mocker.patch( 'feed_file_exporter.gzip_files_in_directory', return_value=([], [SimpleNamespace(output_path='tmp/a.TXT.gz')]), ) with pytest.raises(SystemExit, match='Gzip compression failed.'): ffe.prepare_s3_transfer_files( generated_files=['tmp/a.TXT'], temp_file_path='tmp/out', gzip_enabled=True, continue_on_error=False, ) @pytest.mark.parametrize( ('transfer_mode', 'expects_individual', 'expects_zip'), [ ('Individual Files', True, False), ('Zip File', False, True), ('Both', True, True), ('None', False, False), ('Unexpected', False, False), ], ) def test_resolve_s3_transfer_mode( transfer_mode, expects_individual, expects_zip, ): """S3 transfer mode parsing stays local and deterministic.""" assert ffe.resolve_s3_transfer_mode(transfer_mode) == ( expects_individual, expects_zip, ) def test_build_zipfile_basename_uses_sap_ticket_naming(): """SAP Settlement zip name follows the INT-2486 format.""" result = ffe.build_zipfile_basename( temp_file_path='output/20260310/sap_settlement', proc_mode='SAP_SETTLEMENT', current_time=ffe.datetime(2026, 3, 10, 9, 30, 0), ) assert result == 'sony_settlement_feed-2026_03_10' def test_build_zipfile_basename_preserves_default_for_non_sap(): """Non-SAP zip names continue to use the output folder basename.""" result = ffe.build_zipfile_basename( temp_file_path='output/20260310/us/', proc_mode='US', current_time=ffe.datetime(2026, 3, 10, 9, 30, 0), ) assert result == 'us' def test_upload_files(mocker, test_file_name_list): """Test upload_files() function.""" file_count = 3 mock_boto = mocker.patch( 'src.utils.s3_backoff_utils.s3_resource.Bucket', return_value=MagicMock(upload_file=MagicMock()) ) # mock_boto = mocker.patch('boto3.resource', MagicMock()) # mock_upload = mocker.patch('s3_bucket.upload_file' output_file_list = test_file_name_list(file_count) tmp_dir = tempfile.TemporaryDirectory() upload_files(output_file_list, tmp_dir.name, "tmp/upload", "bucket_name") calls = mock_boto.return_value.upload_file.call_count assert calls == file_count def test_split_rows_to_files(): """Test split_rows_to_files() method.""" # TODO - Incomplete pass def test_split_res_to_files(): """Test split_res_to_files() method.""" # TODO - Incomplete pass