"""Lambda sr-delivery-youtube function module."""
import json
import os
from datetime import datetime
from datetime import timezone
import random
import sentry_sdk
from sentry_sdk.integrations.aws_lambda import AwsLambdaIntegration
from src.connectors import sftp_asset_delivery
from src.utils.typedload import get_loader
from .common import logger
from .common.connectors import s3_assets
from .common.connectors import s3_delivery_audit
from .common.connectors import s3_sound_recordings
from soundrecording_utils.constants.ddex.constants import RuleService
from soundrecording_utils.metadata.types import Asset
from soundrecording_utils.metadata.types import OrchardSoundRecording
from soundrecording_utils.ddex.generate import generate_ddex
import config
as_bytes = 0
destination_subdir = 1
# initialize sentry
sentry_dsn = os.environ.get(
'SENTRY_DSN', config.secrets_manager_client.get_cred('SENTRY_DSN'))
if sentry_dsn:
logger.info('Initializing with sentry')
sentry_sdk.init(
sentry_dsn,
integrations=[AwsLambdaIntegration()]
)
else:
logger.info('Initializing without sentry')
service = RuleService.YOUTUBE
class NoAssets(Exception):
"""No deliverable assets on an OrchardSoundRecording."""
pass
def get_osr(sound_recording_id: str, version_id: str | None = None) -> OrchardSoundRecording: # noqa:E501
"""Get the metadata blob from the event message."""
# retrieve sound recording from s3
if version_id:
(raw_data, _) = s3_sound_recordings.get_sound_recording_version(
sound_recording_id,
version_id
)
else:
(_, raw_data, _) = s3_sound_recordings.get_latest_sound_recording_version( # noqa:E501
sound_recording_id
)
json_data = json.loads(raw_data)
loader = get_loader()
osr = loader.load(json_data, OrchardSoundRecording)
return osr
def _generate_batch_id() -> str:
date_str = datetime.now().strftime('%Y%m%d%H%M%S')
random_str = str(random.randint(0, 9999)).rjust(4, '0')
return date_str + random_str
def _s3_remote_foldernames(batch_id, isrc):
resources_dirname = config.RESOURCES_DIRNAME.rstrip('/')
return (
batch_id,
batch_id + '/' + isrc,
batch_id + '/' + isrc + '/' + resources_dirname,
)
def _select_upload_asset(
sound_recording_metadata: OrchardSoundRecording
) -> Asset:
"""Select file to upload.."""
for extension in config.ALLOWED_ASSET_TYPES:
assets = [
x for x in sound_recording_metadata.assets
if (x.extension if isinstance(x, Asset) else x.get('extension', '')) == extension
]
if assets:
return assets[0]
raise NoAssets()
def _transfer_files_to_youtube(
xml_files: list,
assets: list,
upload_asset: Asset | None,
execution_name: str
) -> None:
"""Transfer DDEX XML and assets to YouTube via SFTP.
Args:
xml_files (list): List of tuples containing XML file bytes and their
respective subdirectory.
assets (list): List of tuples containing asset file bytes and their
respective subdirectory.
upload_asset (Asset | None): The asset to be uploaded, if any.
execution_name (str): The step function execution ID.
The final entry in xml_files is assumed to be the "Batch Complete" XML.
"""
for byte_data, subdir in xml_files[:-1]:
sftp_asset_delivery.write_file(
file_bytes=byte_data,
subdir=subdir,
step_function_execution_id=execution_name
)
# Transfer actual asset to YouTube via SFTP
if upload_asset and assets[0][as_bytes] is not None:
sftp_asset_delivery.write_file(
file_bytes=assets[0][as_bytes],
subdir=assets[0][destination_subdir],
step_function_execution_id=execution_name
)
# Transfer "Batch Complete" XML to YouTube via SFTP
batch_complete = xml_files[-1]
sftp_asset_delivery.write_file(
file_bytes=batch_complete[as_bytes],
subdir=batch_complete[destination_subdir],
step_function_execution_id=execution_name
)
def handler(event, context):
"""Lambda entry point."""
try:
takedown: bool = event.get('delivery_type', '') == 'TAKEDOWN_DELIVERY'
upload = event['upload_asset'] and config.PROCESS_ASSET
assert type(upload) == bool # noqa:E721
sound_recording_id: str = event['sound_recording']['id']
assert type(sound_recording_id) == str # noqa:E721
version_id: str = event['sound_recording'].get('version')
assert type(version_id) == str or version_id is None # noqa:E721
osr: OrchardSoundRecording = get_osr(
sound_recording_id,
version_id
)
batch_id = _generate_batch_id()
execution_metadata = {
'timestamp': datetime.now(timezone.utc),
'message_thread_id': 1,
'message_id': batch_id,
}
# select asset by metadata if upload required
upload_asset = _select_upload_asset(osr) if upload else None
# generate XML based on metadata and asset
ddex: str = generate_ddex(
osr, upload_asset, execution_metadata, config.APPLICATION_NAME,
service, selected_version=config.DDEX_FILE_VERSION, takedown=takedown
)
# fetch actual audio asset
asset_bytes = s3_assets.download_asset(
upload_asset.filename if isinstance(upload_asset, Asset) else upload_asset.get('filename'), # noqa:E501
upload_asset.extension if isinstance(upload_asset, Asset) else upload_asset.get('extension') # noqa:E501
) if upload_asset else None
# list files to transfer
sr_isrc = osr.isrc
(batch_dir, isrc_dir, resources_dir) = _s3_remote_foldernames(batch_id, sr_isrc) # noqa:E501
assets = [
(
asset_bytes,
f'{resources_dir}/{upload_asset.filename}.{upload_asset.extension}' # noqa:E501
)
] if upload_asset else []
xml_files = []
seen_isrcs = set()
# YouTube requires one XML per ISRC. The DDEX is the same except for the ISRC value
for t in osr.track_connection.tracks:
if t.isrc != osr.isrc and t.isrc not in seen_isrcs:
logger.debug(f'Adding XML to ref ISRC ${osr.isrc} for {t.isrc}') # noqa:E501
ddex_copy = ddex.replace(f'{osr.isrc}', f'{t.isrc}') # noqa:E501
xml_files.append(
(
ddex_copy.encode('utf-8'),
f'{batch_id}/{t.isrc}/{t.isrc}.xml'
)
)
seen_isrcs.add(t.isrc)
# add primary ISRC XML and batch complete XML
primary_isrc = (
ddex.encode('utf-8'),
f'{isrc_dir}/{sr_isrc}.xml'
)
batch_complete = (
b'',
f'{batch_dir}/delivery.complete'
)
xml_files.append(
primary_isrc
)
xml_files.append(
batch_complete
)
# upload XML file to S3 for future auditing
# @todo For now log in the normal way. Later we may want to store each
# XML separately.
s3_delivery_audit.write_sr_delivery_audit_xml(
local=primary_isrc[0],
remote=primary_isrc[1],
step_function_execution_id=event['execution_name'],
service=service.value
)
# transfer files to YouTube via SFTP
_transfer_files_to_youtube(
xml_files,
assets,
upload_asset,
event['execution_name']
)
assets += xml_files
return {
'details': {
'batch_id': batch_id,
'filenames': [x[destination_subdir] for x in assets]
}
}
# suppress entry, fail lambda function
except NoAssets as e:
sentry_sdk.init()
raise e
except Exception as e:
logger.exception(str(e))
raise e