"""Unit tests for Setlive workflow tasks.""" from datetime import datetime import re from unittest.mock import MagicMock from unittest.mock import patch import pytest from feed_ingestion.flows.setlive import config from feed_ingestion.flows.setlive import tasks _date = '2024-01-01' _report_name = 'opt_ins' class TestBootstrap(object): """Test bootstrap.""" @pytest.fixture def expected_bootstrap_response(self): """Response for a bootstrap task.""" _date_dt = datetime.strptime(_date, '%Y-%m-%d').date() filename = config.reports[_report_name]['filename_template'].format( date=_date_dt ) file_pattern = rf'.*\/{filename}' return { 'feed_name': '_'.join([config.feed_name, _report_name]), 'date': _date, 'report_name': _report_name, 'secrets_path': config.secrets_path, 'drop_bucket': config.drop_bucket, 'drop_path': config.drop_path, 's3_dir_path': f's3://{config.drop_bucket}/{config.drop_path}', 'file_pattern': file_pattern, 'snowflake_pattern': filename, 'staging_raw_table': config.reports[_report_name][ 'staging_raw_table'], 'staging_raw_table_errors': config.reports[_report_name][ 'staging_raw_table_errors'], 'pii_columns': config.reports[_report_name]['pii_columns'], } @patch('feed_ingestion.flows.setlive.tasks.garcon_feed_status') def test_bootstrap(self, feed_status_mock, expected_bootstrap_response): """Test bootstrap task.""" context = { 'activity': MagicMock(), 'date': _date, 'report': _report_name, 'reload': None, } result = tasks.bootstrap(**context) assert result == expected_bootstrap_response def test_filename_template_matches_current_vendor_format(): """The configured pattern should match the observed Setlive filename.""" report_date = datetime.strptime('2026-05-05', '%Y-%m-%d').date() pattern = config.reports['opt_ins']['filename_template'].format( date=report_date ) assert re.match( pattern, 'artistGroupId-2026-05-05T12-00-00-000Z.csv', ) assert not re.match( pattern, 'artistGroupId-2026-05-06T12-00-00-000Z.csv', ) @patch('feed_ingestion.flows.setlive.tasks.SetliveSnowflakeExecutor') def test_load_staging_raw_table_reports_typical(mock_executor): """Test typical staging raw load sequence.""" activity = MagicMock() feed_name = 'setlive_opt_ins' date = '2024-01-01' report_name = 'opt_ins' s3_dir_path = 's3://bucket/path' staging_raw_table_name = 'staging_table' staging_raw_errors_table_name = 'staging_table_errors' file_names = {'files': ['file1.csv', 'file2.csv']} snowflake_pattern = '.*-2024-01-01T[^/]*\\.csv$' pii_columns = [ 'EMAIL', 'FIRST_NAME', 'LAST_NAME', 'POSTAL_CODE', 'PHONE_NUMBER', ] mock_exec = mock_executor.return_value.__enter__.return_value tasks.load_staging_raw_table_reports( activity, feed_name, date, report_name, s3_dir_path, staging_raw_table_name, staging_raw_errors_table_name, file_names, snowflake_pattern, pii_columns, ) mock_exec.create_temp_staging_raw_table.assert_called_once_with( report_name=report_name, temp_staging_raw_table='temp_staging_table_20240101', ) mock_exec.assign_masking_on_temp_staging_raw_table.assert_called_once_with( temp_staging_raw_table='temp_staging_table_20240101', pii_columns=pii_columns, ) mock_exec.create_stage.assert_called_once_with( s3_dir_path=s3_dir_path, stage_name='stage_setlive_20240101', ) mock_exec.load_temp_staging_raw_table.assert_called_once_with( temp_staging_raw_table='temp_staging_table_20240101', file_name='.*(file1\\.csv|file2\\.csv)$', stage_name='stage_setlive_20240101', ) mock_exec.clean_staging_raw_table.assert_called_once_with( staging_raw_table=staging_raw_table_name, staging_raw_errors_table=staging_raw_errors_table_name, file_name='^(file1\\.csv|file2\\.csv)$', date=date, ) mock_exec.load_staging_raw_table.assert_called_once_with( staging_raw_table=staging_raw_table_name, staging_raw_errors_table=staging_raw_errors_table_name, temp_staging_raw_table='temp_staging_table_20240101', report_name=report_name, date=date, file_name='^(file1\\.csv|file2\\.csv)$', ) mock_exec.drop_table.assert_called_once_with( table='temp_staging_table_20240101', ) mock_exec.drop_stage.assert_called_once_with( stage_name='stage_setlive_20240101', ) @patch('feed_ingestion.flows.setlive.tasks.task_status') @patch('feed_ingestion.flows.setlive.tasks.common_check_files_on_s3') def test_mark_ingested_files(mock_check_files_on_s3, mock_task_status): """Test that matched files are persisted in task status.""" mock_check_files_on_s3.return_value = { 'source_files_dict': { 'files': ['artistGroupId-2026-05-05T12-00-00-000Z.csv'] } } tasks.mark_ingested_files( activity=MagicMock(), feed_name='setlive_opt_ins', date='2026-05-05', s3_download_path='s3://prod-orcdbucket/feed-drop/SetLive/', file_pattern=r'.*\/.*-2026-05-05T[^/]*\.csv$', ) mock_task_status.set_values.assert_any_call( 'setlive_opt_ins', '2026-05-05', 'ingested_files_status', ['artistGroupId-2026-05-05T12-00-00-000Z.csv'], ) mock_task_status.set_values.assert_any_call( 'setlive_opt_ins', '2026-05-05', 'ingested_files', ['artistGroupId-2026-05-05T12-00-00-000Z.csv'], ) @patch('feed_ingestion.flows.setlive.tasks.common_check_files_on_s3') def test_check_files_on_s3_returns_stop_payload(mock_check_files_on_s3): """Test stop responses are passed through unchanged.""" mock_check_files_on_s3.return_value = {'stop': True} result = tasks.check_files_on_s3( activity=MagicMock(), feed_name='setlive_opt_ins', date='2026-05-05', s3_download_path='s3://prod-orcdbucket/feed-drop/SetLive/', file_pattern=r'.*\/.*-2026-05-05T[^/]*\.csv$', list_output=True, ) assert result == {'stop': True} @patch('feed_ingestion.flows.setlive.tasks.task_status') @patch('feed_ingestion.flows.setlive.tasks.common_check_files_on_s3') def test_check_files_on_s3_returns_only_new_files( mock_check_files_on_s3, mock_task_status, ): """Only newly discovered files should be passed to the load step.""" mock_check_files_on_s3.return_value = { 'source_files_dict': { 'files': [ 'existing-2026-05-09T19-37-34-434Z.csv', 'new-2026-05-09T19-37-34-434Z.csv', ] } } mock_task_status.get_values.return_value = [ 'existing-2026-05-09T19-37-34-434Z.csv' ] result = tasks.check_files_on_s3( activity=MagicMock(), feed_name='setlive_opt_ins', date='2026-05-09', s3_download_path='s3://prod-orcdbucket/feed-drop/SetLive/', file_pattern=r'.*\/.*-2026-05-09T[^/]*\.csv$', list_output=True, ) assert result == { 'source_files_dict': { 'files': ['new-2026-05-09T19-37-34-434Z.csv'] } }