import os from datetime import datetime from unittest.mock import patch import config import index import pytest from constants import queries from freezegun import freeze_time from tests.unit.test_utils import assert_ingestion_not_started, \ assert_ingestion_started, DdexQueryTest logger = config.get_current_logger(config.app_logger) class TestOriginalReleaseDatetimeTime(DdexQueryTest): def test_scenario_1_1(self): """Scenario 1.1 Now = 2025-06-05T10:00.000Z , One delivery with is_confidential = false, original_release_datetime='2025-06-05' Now is after the datetime we want to ingest product, so we should ingest this product. """ self.mock_myql_db_time(datetime(2025, 6, 5, 10, 00, 00)) s3_key_name = 'test_scenario_1_1' self.insert_ddex_delivery(s3_key_name, original_release_datetime='2025-06-05') rows_to_ingest = index.get_ddex_delivery_to_ingest(10, False) assert len([x for x in rows_to_ingest if s3_key_name == x.get('s3_key_name')]) == 1 def test_scenario_1_2(self): """Scenario 1.2 - Now = 2025-06-05T10:00.000Z , One delivery with is_confidential = false, original_release_datetime = ‘2025-06-08’ Now is before the datetime we want to ingest product, so we should NOT ingest this product. """ self.mock_myql_db_time(datetime(2025, 6, 5, 10, 00, 00)) s3_key_name = 'test_scenario_1_2' self.insert_ddex_delivery(s3_key_name, original_release_datetime='2025-06-08') rows_to_ingest = index.get_ddex_delivery_to_ingest(10, False) assert len([x for x in rows_to_ingest if s3_key_name == x.get('s3_key_name')]) == 0 class TestHasArtworkProcessingOrder(DdexQueryTest): @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2_1(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """ Scenario 2.1 - One initial delivery with artwork, 2 later deliveries with xml only delivery 1, has_artwork = true delivery 2, has_artwork = false delivery 3, has_artwork = false Desired outcome: ingest delivery 1, only when that has status = ingest_succeeded do we want to ingest delivery 3. We do NOT want to ingest delivery 2 as it has no artwork and is not the most recent delivery. """ mock_get_full_execution_count_by_arn.return_value = 0 product = 'TESTGRID1' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' ddex_delivery_2 = f'test/{product}_20250102/{product}.xml' ddex_delivery_3 = f'test/{product}_20250103/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, has_artwork=True, original_release_datetime='2051-06-05') self.insert_ddex_delivery(ddex_delivery_2, has_artwork=True, original_release_datetime='2051-06-05') self.insert_ddex_delivery(ddex_delivery_3, has_artwork=False, original_release_datetime='2020-06-05') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_2, artwork_ingestion_only=True) assert_ingestion_not_started(mock_start_execution_by_arn, ddex_delivery_1) assert_ingestion_not_started(mock_start_execution_by_arn, ddex_delivery_3) self.set_ddex_delivery_to_ingest_succeeded( ddex_delivery_2) index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_3 ) assert_ingestion_not_started(mock_start_execution_by_arn, ddex_delivery_1) self.set_ddex_delivery_to_ingest_succeeded( ddex_delivery_3) index.handler({}, {}) assert_ingestion_not_started(mock_start_execution_by_arn, ddex_delivery_1) @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2_3(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """ Scenario 2.3 - Multiple artwork deliveries and multiple xml only delivery 1, has_artwork = true delivery 2, has_artwork = false delivery 3, has_artwork = true delivery 4, has_artwork = false delivery 5, has_artwork = false Desired outcome: ingest delivery 3 (latest with artwork), only when that has status = ingest_succeeded do we want to ingest delivery 5. We do NOT want to ingest delivery 1, 2, 4 as it they have all been superseded by later deliveries. """ mock_get_full_execution_count_by_arn.return_value = 0 product = 'TESTGRID3' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' ddex_delivery_2 = f'test/{product}_20250102/{product}.xml' ddex_delivery_3 = f'test/{product}_20250103/{product}.xml' ddex_delivery_4 = f'test/{product}_20250104/{product}.xml' ddex_delivery_5 = f'test/{product}_20250105/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, has_artwork=True, original_release_datetime='2051-06-05') self.insert_ddex_delivery(ddex_delivery_2, has_artwork=False, original_release_datetime='2051-06-05') self.insert_ddex_delivery(ddex_delivery_3, has_artwork=True, original_release_datetime='2020-06-05') self.insert_ddex_delivery(ddex_delivery_4, has_artwork=False, original_release_datetime='2020-06-05') self.insert_ddex_delivery(ddex_delivery_5, has_artwork=False, original_release_datetime='2020-06-05') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_3, artwork_ingestion_only=True) assert_ingestion_not_started(mock_start_execution_by_arn, [ddex_delivery_1, ddex_delivery_2, ddex_delivery_4, ddex_delivery_5]) self.set_ddex_delivery_to_ingest_succeeded( ddex_delivery_3) index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_3, artwork_ingestion_only=True) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_5, artwork_ingestion_only=False) assert_ingestion_not_started(mock_start_execution_by_arn, [ddex_delivery_1, ddex_delivery_2, ddex_delivery_4]) @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2_4(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Scenario 2.4 - One delivery with artwork and xml delivery 1, has_artwork = true Desired outcome: ingest delivery 1 as it has both artwork and the latest xml.""" mock_get_full_execution_count_by_arn.return_value = 0 self.insert_ddex_delivery('test/TESTGRID4_20250101/TESTGRID4.xml', has_artwork=True, original_release_datetime='2023-06-05') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, 'test/TESTGRID4_20250101/TESTGRID4.xml', artwork_ingestion_only=False) @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2_5(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """ Scenario 2.5 - Multiple deliveries with artwork and xml, latest delivery has artwork delivery 1, has_artwork = true delivery 2, has_artwork = false delivery 3, has_artwork = true Desired outcome: ingest delivery 3 as it has both latest artwork and the latest xml. """ mock_get_full_execution_count_by_arn.return_value = 0 product = 'TESTGRID5' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' ddex_delivery_2 = f'test/{product}_20250102/{product}.xml' ddex_delivery_3 = f'test/{product}_20250103/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, has_artwork=True, original_release_datetime='2023-06-05') self.insert_ddex_delivery(ddex_delivery_2, has_artwork=False, original_release_datetime='2023-06-05') self.insert_ddex_delivery(ddex_delivery_3, has_artwork=True, original_release_datetime='2023-06-05') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_3) assert_ingestion_not_started(mock_start_execution_by_arn, [ddex_delivery_1, ddex_delivery_2]) @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2_6(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """ Scenario 2.6 - Multiple deliveries with xml only, no delivery has artwork delivery 1, has_artwork = false delivery 2, has_artwork = false delivery 3, has_artwork = false Desired outcome: ingest delivery 3 as it has latest xml and there is no delivery with artwork. """ product = 'TESTGRID6' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' ddex_delivery_2 = f'test/{product}_20250102/{product}.xml' ddex_delivery_3 = f'test/{product}_20250103/{product}.xml' mock_get_full_execution_count_by_arn.return_value = 0 self.insert_ddex_delivery(ddex_delivery_1, has_artwork=False, original_release_datetime='2023-06-05') self.insert_ddex_delivery(ddex_delivery_2, has_artwork=False, original_release_datetime='2023-06-05') self.insert_ddex_delivery(ddex_delivery_3, has_artwork=False, original_release_datetime='2023-06-05') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_3) assert_ingestion_not_started(mock_start_execution_by_arn, [ddex_delivery_1, ddex_delivery_2] ) class TestState(DdexQueryTest): @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_execution_not_start_if_export_in_progress(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'GRIDEXECUTIONSTATUS' self.insert_ddex_delivery(f'test/{test_GRID}_20250101/{test_GRID}.xml', has_artwork=True, status='ingest_started', original_release_datetime='2023-06-05') self.insert_ddex_delivery(f'test/{test_GRID}_20250102/{test_GRID}.xml', has_artwork=False, original_release_datetime='2023-06-05') index.handler({}, {}) assert_ingestion_not_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250102/{test_GRID}.xml') self.set_ddex_delivery_to_ingest_succeeded( f'test/{test_GRID}_20250101/{test_GRID}.xml') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250102/{test_GRID}.xml') class TestIsConfidential(DdexQueryTest): @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_ddex_start_for_confidential_data(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Start date is before now(), so processing should be started.""" mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'GRIDCONFIDENTIAL' self.insert_ddex_delivery(f'test/{test_GRID}_20250101/{test_GRID}.xml', is_confidential=True, original_release_datetime='2023-06-05 00:00:00') index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml') self.insert_ddex_delivery(f'test/{test_GRID}_20250102/{test_GRID}.xml', is_confidential=False, original_release_datetime='2023-06-05') self.set_ddex_delivery_to_ingest_succeeded( f'test/{test_GRID}_20250101/{test_GRID}.xml') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250102/{test_GRID}.xml') assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml') @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2_7(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """ Scenario 2.7 - Multiple deliveries with xml only delivery 1, has_artwork = true, is_confidential=true delivery 2, has_artwork = false delivery 3, has_artwork = false Desired outcome: ingest delivery 3 as it has latest xml and there is no non confidentialdelivery with artwork . """ product = 'TESTGRID7' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' ddex_delivery_2 = f'test/{product}_20250102/{product}.xml' ddex_delivery_3 = f'test/{product}_20250103/{product}.xml' mock_get_full_execution_count_by_arn.return_value = 0 self.insert_ddex_delivery(ddex_delivery_1, has_artwork=False, is_confidential=True, original_release_datetime='2023-06-05') self.insert_ddex_delivery(ddex_delivery_2, has_artwork=False, original_release_datetime='2023-06-05') self.insert_ddex_delivery(ddex_delivery_3, has_artwork=False, original_release_datetime='2023-06-05') index.handler({}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_3, ) @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') @freeze_time("2025-06-05 10:00:00") def test_ddex_confidential_start(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test that processing for confidential will start if earliest_start_datetime is in one hour""" # noqa mock_get_full_execution_count_by_arn.return_value = 0 self.mock_myql_db_time(datetime(2025, 6, 5, 10, 00, 00)) earliest_start_datetime = '2025-06-05 00:00:00 ' test_GRID = 'GRIDCONFIDENTIAL_1' self.insert_ddex_delivery(f'test/{test_GRID}_20250101/{test_GRID}.xml', is_confidential=True, original_release_datetime=earliest_start_datetime) index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml') @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') @freeze_time("2025-06-05 10:00:00") def test_ddex_confidential_process_confidential_flag(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test process_confidential flag""" # noqa mock_get_full_execution_count_by_arn.return_value = 0 self.mock_myql_db_time(datetime(2025, 6, 5, 10, 00, 00)) earliest_start_datetime = '2025-06-05 00:00:00 ' test_GRID = 'GRIDCONFIDENTIAL_1' self.insert_ddex_delivery(f'test/{test_GRID}_20250101/{test_GRID}.xml', is_confidential=True, original_release_datetime=earliest_start_datetime) index.handler({'PROCESS_CONFIDENTIAL': False}, {}) assert_ingestion_not_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml') @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') @freeze_time("2025-06-05 08:00:10") def test_ddex_confidential_dont_start(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test tha processing for confidential will start if earliest_start_datetime is not in one hour""" # noqa mock_get_full_execution_count_by_arn.return_value = 0 self.mock_myql_db_time(datetime(2025, 6, 5, 8, 0, 10)) earliest_start_datetime = '2025-06-05T09:00:20.000Z' test_GRID = 'GRIDCONFIDENTIAL_2' ddex_delivery_1 = f'test/{test_GRID}_20250101/{test_GRID}.xml' self.insert_ddex_delivery(ddex_delivery_1, is_confidential=True, original_release_datetime=earliest_start_datetime) index.handler({}, {}) assert_ingestion_not_started(mock_start_execution_by_arn, ddex_delivery_1) class TestCheckIsConfidentialProject(DdexQueryTest): @freeze_time("2012-01-14 03:21:34") @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_1(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn ): """Test in case of mistake in query confidential project won't be processed because current time @freeze_time("2012-01-14 03:21:34") is before earliest_start_date='2024-01-01' """ product = 'TESTCONFIDENTIAL1' mock_get_full_execution_count_by_arn.return_value = 0 queries.CONFIDENTIAL_DDEX_DELIVERIES = \ 'select * from ddex_ingester.ddex_delivery_to_ingest_query_test' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, is_confidential=True, original_release_datetime='2024-01-01' ) with pytest.raises(Exception) as excinfo: index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert str( excinfo.value) == "Attempt to process confidential product BEFORE" \ " earliest_datetime , now = 2012-01-14 03:21:34" \ " original_release_datetime = 2024-01-01 00:00:00" \ " s3 Bucket: None," \ " s3 Key : test/TESTCONFIDENTIAL1_20250101/TESTCONFIDENTIAL1.xml" @freeze_time("2012-01-14 03:21:34") @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_2(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn ): """Test check won't fall for correct data : current time @freeze_time(("2012-01-14 03:21:34") is equal to earliest_start_date='2012-01-14 03:21:24' """ product = 'TESTCONFIDENTIAL2' mock_get_full_execution_count_by_arn.return_value = 0 queries.CONFIDENTIAL_DDEX_DELIVERIES = \ 'select * from ddex_ingester.ddex_delivery_to_ingest_query_test' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, is_confidential=True, original_release_datetime='2012-01-14 03:21:24' ) index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_1) @freeze_time("2010-01-14 03:21:34") @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_3(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn ): """Test in case of mistake in query confidential project won't be processed because current time @freeze_time("2010-01-14 03:21:34") is before earliest_start_date='2025-01-14' """ product = 'TESTCONFIDENTIAL2' mock_get_full_execution_count_by_arn.return_value = 0 queries.CONFIDENTIAL_DDEX_DELIVERIES = \ 'select * from ddex_ingester.ddex_delivery_to_ingest_query_test' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, is_confidential=True, original_release_datetime='2025-01-14' ) with pytest.raises(Exception) as excinfo: index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert str( excinfo.value) == "Attempt to process confidential product" \ " BEFORE earliest_datetime ," \ " now = 2010-01-14 03:21:34" \ " original_release_datetime = 2025-01-14 00:00:00" \ " s3 Bucket: None," \ f" s3 Key : {ddex_delivery_1}" @freeze_time("2012-10-02 11:55:00") @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_4(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn ): """Test in case of mistake in query confidential project won't be processed because current time @freeze_time("2012-10-02 11:55:00") is before earliest_start_date='2012-10-03' """ product = 'TESTCONFIDENTIAL1' mock_get_full_execution_count_by_arn.return_value = 0 queries.CONFIDENTIAL_DDEX_DELIVERIES = \ 'select * from ddex_ingester.ddex_delivery_to_ingest_query_test' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, is_confidential=True, original_release_datetime='2012-10-03', ) with pytest.raises(Exception) as excinfo: index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert str( excinfo.value) == \ "Attempt to process confidential product BEFORE earliest_datetime ," \ " now = 2012-10-02 11:55:00 " \ "original_release_datetime = 2012-10-03 00:00:00" \ " s3 Bucket: None," \ f" s3 Key : {ddex_delivery_1}" @freeze_time("2012-10-02 11:55:00") @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_6(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn ): """Test that check won't fail for non-confidential products.""" product = 'TESTCONFIDENTIAL1' mock_get_full_execution_count_by_arn.return_value = 0 queries.CONFIDENTIAL_DDEX_DELIVERIES = \ 'select * from ddex_ingester.ddex_delivery_to_ingest_query_test' ddex_delivery_1 = f'test/{product}_20250101/{product}.xml' self.insert_ddex_delivery(ddex_delivery_1, is_confidential=False, original_release_datetime='2015-10-03', ) index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert_ingestion_started(mock_start_execution_by_arn, ddex_delivery_1) class TestArtworkIngestionOnly(DdexQueryTest): @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_artwork_ingestion_only_flag_1(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test scenario: There are 2 confidential ddex ingestions for the same grid. First one with artwork , Second without Expected behaviour: first ingestion will be triggered for firs file with artwork_ingestion_only = True second ingestion will be triggered for second file with artwork_ingestion_only = False """ mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'GRIDEXECUTIONSTATUS' self.insert_ddex_delivery(f'test/{test_GRID}_20250101/{test_GRID}.xml', has_artwork=True, is_confidential=True, original_release_datetime='2023-06-05 00:00:00') self.insert_ddex_delivery(f'test/{test_GRID}_20250102/{test_GRID}.xml', has_artwork=False, is_confidential=True, original_release_datetime='2023-06-05 00:00:00') index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml', artwork_ingestion_only=True) self.set_ddex_delivery_to_ingest_succeeded( f'test/{test_GRID}_20250101/{test_GRID}.xml') index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250102/{test_GRID}.xml') class TestSimultaneousExecutionForTheSameKey(DdexQueryTest): @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') @freeze_time("2025-01-01 00:00:00") def test_simultaneous_execution_for_the_same_key(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test scenario: There are 2 or more scheduler delivery for the same s3_key. Validate that ony one will be triggered per flow control run. """ mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'SIMULTANEOUS_EXECUTION' s3_key = f'{test_GRID}_20250101' self.insert_ddex_delivery(f'test/{s3_key}/{test_GRID}.xml', has_artwork=False, is_confidential=True, original_release_datetime='2023-06-05 00:00:00') self.insert_ddex_delivery(f'test/{s3_key}/{test_GRID}.xml', has_artwork=False, is_confidential=True, original_release_datetime='2023-06-05 00:01:00') index.handler({'PROCESS_CONFIDENTIAL': True}, {}) assert mock_start_execution_by_arn.call_count == 1 @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') @freeze_time("2025-01-01 00:00:00") def test_non_confidential_simultaneous_execution_for_the_same_key(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test scenario: There are 2 or more scheduler delivery for the same s3_key. Validate that ony one will be triggered per flow control run. """ mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'SIMULTANEOUS_EXECUTION' s3_key = f'{test_GRID}_20250101' self.insert_ddex_delivery(f'test/{s3_key}/{test_GRID}.xml', has_artwork=False, is_confidential=False, original_release_datetime='2023-06-05 00:00:00') self.insert_ddex_delivery(f'test/{s3_key}/{test_GRID}.xml', has_artwork=False, is_confidential=False, original_release_datetime='2023-06-05 00:01:00') index.handler({'PROCESS_CONFIDENTIAL': False}, {}) assert mock_start_execution_by_arn.call_count == 1 class TestPurgedRelease(DdexQueryTest): @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_purged_release_ingestion_is_triggered(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test scenario: There is purged release. Validate that purged release is triggered. """ mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'purged_release' s3_key = f'{test_GRID}_20250101' self.insert_ddex_delivery(f'test/{s3_key}/{test_GRID}.xml', original_release_datetime='2023-06-05 00:00:00', status='purged_release' ) index.handler({'PROCESS_CONFIDENTIAL': False}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20250101/{test_GRID}.xml', artwork_ingestion_only=False) @patch('index.sfn.start_execution_by_arn') @patch('index.sfn.get_full_execution_name_list_by_arn') @patch('index.sfn.get_full_execution_count_by_arn') def test_soft_delete_triggered_for_previously_ingested_grid(self, mock_get_full_execution_count_by_arn, mock_get_full_execution_name_list_by_arn, mock_start_execution_by_arn): """Test scenario: Product is soft deleted after Validate that purged release is triggered. """ mock_get_full_execution_count_by_arn.return_value = 0 test_GRID = 'simple_ddex_file' s3_key = f'{test_GRID}_20210605' self.insert_ddex_delivery(f'test/{s3_key}/{test_GRID}.xml', original_release_datetime='2021-06-05 00:00:00', ) index.handler({'PROCESS_CONFIDENTIAL': False}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{test_GRID}_20210605/{test_GRID}.xml', artwork_ingestion_only=False) self.set_ddex_delivery_to_ingest_succeeded( f'test/{test_GRID}_20250101/{test_GRID}.xml') purged_test_GRID = 'purged_release' purged_s3_key = f'{purged_test_GRID}_20210605' self.insert_ddex_delivery( f'test/{purged_s3_key}/{purged_test_GRID}.xml', original_release_datetime='2021-06-05 00:00:00', ) index.handler({'PROCESS_CONFIDENTIAL': False}, {}) assert_ingestion_started(mock_start_execution_by_arn, f'test/{purged_test_GRID}_20210605/{purged_test_GRID}.xml', artwork_ingestion_only=False)