"""Lambda sr-delivery-youtube function module.""" import json import os from datetime import datetime from datetime import timezone import random import sentry_sdk from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration from src.connectors import sftp_asset_delivery from src.utils.typedload import get_loader from .common import logger from .common.connectors import s3_assets from .common.connectors import s3_delivery_audit from .common.connectors import s3_sound_recordings from soundrecording_utils.constants.ddex.constants import RuleService from soundrecording_utils.metadata.types import Asset from soundrecording_utils.metadata.types import OrchardSoundRecording from soundrecording_utils.ddex.generate import generate_ddex import config as_bytes = 0 destination_subdir = 1 # initialize sentry sentry_dsn = os.environ.get( 'SENTRY_DSN', config.secrets_manager_client.get_cred('SENTRY_DSN')) if sentry_dsn: logger.info('Initializing with sentry') sentry_sdk.init( sentry_dsn, integrations=[AwsLambdaIntegration()] ) else: logger.info('Initializing without sentry') service = RuleService.YOUTUBE class NoAssets(Exception): """No deliverable assets on an OrchardSoundRecording.""" pass def get_osr(sound_recording_id: str, version_id: str | None = None) -> OrchardSoundRecording: # noqa:E501 """Get the metadata blob from the event message.""" # retrieve sound recording from s3 if version_id: (raw_data, _) = s3_sound_recordings.get_sound_recording_version( sound_recording_id, version_id ) else: (_, raw_data, _) = s3_sound_recordings.get_latest_sound_recording_version( # noqa:E501 sound_recording_id ) json_data = json.loads(raw_data) loader = get_loader() osr = loader.load(json_data, OrchardSoundRecording) return osr def _generate_batch_id() -> str: date_str = datetime.now().strftime('%Y%m%d%H%M%S') random_str = str(random.randint(0, 9999)).rjust(4, '0') return date_str + random_str def _s3_remote_foldernames(batch_id, isrc): resources_dirname = config.RESOURCES_DIRNAME.rstrip('/') return ( batch_id, batch_id + '/' + isrc, batch_id + '/' + isrc + '/' + resources_dirname, ) def _select_upload_asset( sound_recording_metadata: OrchardSoundRecording ) -> Asset: """Select file to upload..""" for extension in config.ALLOWED_ASSET_TYPES: assets = [ x for x in sound_recording_metadata.assets if (x.extension if isinstance(x, Asset) else x.get('extension', '')) == extension ] if assets: return assets[0] raise NoAssets() def _transfer_files_to_youtube( xml_files: list, assets: list, upload_asset: Asset | None, execution_name: str ) -> None: """Transfer DDEX XML and assets to YouTube via SFTP. Args: xml_files (list): List of tuples containing XML file bytes and their respective subdirectory. assets (list): List of tuples containing asset file bytes and their respective subdirectory. upload_asset (Asset | None): The asset to be uploaded, if any. execution_name (str): The step function execution ID. The final entry in xml_files is assumed to be the "Batch Complete" XML. """ for byte_data, subdir in xml_files[:-1]: sftp_asset_delivery.write_file( file_bytes=byte_data, subdir=subdir, step_function_execution_id=execution_name ) # Transfer actual asset to YouTube via SFTP if upload_asset and assets[0][as_bytes] is not None: sftp_asset_delivery.write_file( file_bytes=assets[0][as_bytes], subdir=assets[0][destination_subdir], step_function_execution_id=execution_name ) # Transfer "Batch Complete" XML to YouTube via SFTP batch_complete = xml_files[-1] sftp_asset_delivery.write_file( file_bytes=batch_complete[as_bytes], subdir=batch_complete[destination_subdir], step_function_execution_id=execution_name ) def handler(event, context): """Lambda entry point.""" try: takedown: bool = event.get('delivery_type', '') == 'TAKEDOWN_DELIVERY' upload = event['upload_asset'] and config.PROCESS_ASSET assert type(upload) == bool # noqa:E721 sound_recording_id: str = event['sound_recording']['id'] assert type(sound_recording_id) == str # noqa:E721 version_id: str = event['sound_recording'].get('version') assert type(version_id) == str or version_id is None # noqa:E721 osr: OrchardSoundRecording = get_osr( sound_recording_id, version_id ) batch_id = _generate_batch_id() execution_metadata = { 'timestamp': datetime.now(timezone.utc), 'message_thread_id': 1, 'message_id': batch_id, } # select asset by metadata if upload required upload_asset = _select_upload_asset(osr) if upload else None # generate XML based on metadata and asset ddex: str = generate_ddex( osr, upload_asset, execution_metadata, config.APPLICATION_NAME, service, selected_version=config.DDEX_FILE_VERSION, takedown=takedown ) # fetch actual audio asset asset_bytes = s3_assets.download_asset( upload_asset.filename if isinstance(upload_asset, Asset) else upload_asset.get('filename'), # noqa:E501 upload_asset.extension if isinstance(upload_asset, Asset) else upload_asset.get('extension') # noqa:E501 ) if upload_asset else None # list files to transfer sr_isrc = osr.isrc (batch_dir, isrc_dir, resources_dir) = _s3_remote_foldernames(batch_id, sr_isrc) # noqa:E501 assets = [ ( asset_bytes, f'{resources_dir}/{upload_asset.filename}.{upload_asset.extension}' # noqa:E501 ) ] if upload_asset else [] xml_files = [] seen_isrcs = set() # YouTube requires one XML per ISRC. The DDEX is the same except for the ISRC value for t in osr.track_connection.tracks: if t.isrc != osr.isrc and t.isrc not in seen_isrcs: logger.debug(f'Adding XML to ref ISRC ${osr.isrc} for {t.isrc}') # noqa:E501 ddex_copy = ddex.replace(f'{osr.isrc}', f'{t.isrc}') # noqa:E501 xml_files.append( ( ddex_copy.encode('utf-8'), f'{batch_id}/{t.isrc}/{t.isrc}.xml' ) ) seen_isrcs.add(t.isrc) # add primary ISRC XML and batch complete XML primary_isrc = ( ddex.encode('utf-8'), f'{isrc_dir}/{sr_isrc}.xml' ) batch_complete = ( b'', f'{batch_dir}/delivery.complete' ) xml_files.append( primary_isrc ) xml_files.append( batch_complete ) # upload XML file to S3 for future auditing # @todo For now log in the normal way. Later we may want to store each # XML separately. s3_delivery_audit.write_sr_delivery_audit_xml( local=primary_isrc[0], remote=primary_isrc[1], step_function_execution_id=event['execution_name'], service=service.value ) # transfer files to YouTube via SFTP _transfer_files_to_youtube( xml_files, assets, upload_asset, event['execution_name'] ) assets += xml_files return { 'details': { 'batch_id': batch_id, 'filenames': [x[destination_subdir] for x in assets] } } # suppress entry, fail lambda function except NoAssets as e: sentry_sdk.init() raise e except Exception as e: logger.exception(str(e)) raise e