import logging from typing import Any, Dict, Generator import pytest from tests import config from tests.sfn_integration_tests.utils import ( filenames_in_output_bucket, generate_random_uuid, get_music_assets_for_products, get_tracks_for_products, get_vendor_products, get_video_assets_for_products, merge_assets, stored_checksum_for_asset, ) STEP_FUNCTION_NAME: str = config.BULK_ASSET_DOWNLOAD_STEP_FUNCTION SFN_TIMEOUT: int = 900 S3_BUCKET: str = config.BULK_ASSET_DOWNLOAD_BUCKET @pytest.mark.parametrize('step_function_client', [STEP_FUNCTION_NAME], indirect=True) def test_bulk_asset_download_sfn( api_client_with_token: Any, step_function_client: Any, s3_client: Any, export_id: str, api_client: Any, teardown: Any, s3_filenames_holder: Dict[str, Any], ) -> None: """Test the bulk asset download step function. Test uses a seeded vendor (ID 793968) copied from Production. Products 6146538 & 6149727 are Digital Audio singles; Products 6149736 & 6149748 are Digital Video singles. Assets for 6146538, 6149736 and 6149748 are replicated to QA (CDAM-3519). Assets for 6149727 remain in Production to test error reporting (AUT-890). """ vendor_id: int = 793968 # Step 1: Fetch vendor product data from multiple OWS services product_ids = get_vendor_products(api_client, vendor_id) track_info = get_tracks_for_products(api_client_with_token, product_ids) music_assets = get_music_assets_for_products( api_client_with_token, product_ids, config.AUDIO_ASSET_TYPES ) video_assets = get_video_assets_for_products( api_client_with_token, product_ids, config.VIDEO_ASSET_TYPES ) # Step 2: Combine all asset types and generate expected output filenames assets_by_product = merge_assets(music_assets, video_assets) # Calculate stored checksum for all assets in the input bucket (qa-orcd-mezzanine-assets) stored_checksum_for_all_assets = stored_checksum_for_asset( s3_client, assets_by_product ) # Get expected filenames to verify in the bucket s3_filenames = filenames_in_output_bucket(track_info, assets_by_product) # Step 3: Store filenames in holder so teardown fixture can access them for cleanup s3_filenames_holder['value'] = s3_filenames # Step 4: Execute the bulk asset download step function with test parameters input_data: Dict[str, Any] = { 'account_type': 'vendor', 'account_id': vendor_id, 'asset_types': ['VIDEO_MASTER', 'WAV', 'TIF'], 'export_id': export_id, } step_function_client.execute( input_data, assert_success=True, timeout=SFN_TIMEOUT, execution_name=export_id ) # Step 5: Verify that step function execution completed successfully # An error report should be generated since this vendor has assets in Production bucket error_report_key = f'{export_id}/error_report.csv' s3_client.assert_object_exists(S3_BUCKET, error_report_key) # Step 6: Verify that assets in QA bucket were copied to output bucket with correct filenames and checksums # Build a (product_id, tuid) → expected_checksum lookup checksum_lookup = { (product_id, asset['tuid']): asset['checksum'] for product_id, assets in stored_checksum_for_all_assets.items() for asset in assets } for product_id, assets in s3_filenames.items(): for asset in assets: tuid = asset['tuid'] filename = asset['filename'] expected_checksum = checksum_lookup.get((product_id, tuid)) s3_key = f'{export_id}/{filename}' s3_client.assert_object_exists(S3_BUCKET, s3_key) s3_client.verify_s3_object_sha256(S3_BUCKET, s3_key, expected_checksum) @pytest.fixture def export_id() -> str: """ Fixture that generates a unique export ID used as the Step Function execution name and as the S3 key prefix in output bucket. """ return f'TestBulkDownloader-{generate_random_uuid()}' @pytest.fixture def s3_filenames_holder() -> Dict[str, Any]: """ Provide a mutable container to share S3 filename data between test execution and teardown cleanup. This allows the teardown fixture to know which files to delete after the test completes. """ return {} @pytest.fixture def teardown( s3_client: Any, export_id: str, s3_filenames_holder: Dict[str, Any] ) -> Generator[None, None, None]: """ Clean up all S3 resources created during the test to prevent storage accumulation. This fixture runs after the test completes (both on success and failure). """ yield logging.info( f"Starting cleanup of output artifacts data in bucket '{S3_BUCKET}' for Execution: {export_id}" ) # Collect all S3 keys that need to be deleted keys_to_delete = [f'{export_id}/error_report.csv'] # Add all asset files that were copied during the test for filenames in s3_filenames_holder.get('value', {}).values(): keys_to_delete.extend(f'{export_id}/{file["filename"]}' for file in filenames) # Delete each object individually with error handling to ensure cleanup continues for s3_key in keys_to_delete: logging.info(f'Deleting S3 object: {s3_key}') try: s3_client.delete_if_object_present(S3_BUCKET, s3_key) except Exception as e: logging.warning(f'Failed to delete {s3_key}: {e}')