"""Metadata logic.""" import re from datetime import datetime from typing import Any from zoneinfo import ZoneInfo from flask import g from gtin.validator import is_valid_GTIN from requests.exceptions import HTTPError from sqlalchemy import func from sqlalchemy.exc import SQLAlchemyError from video.connectors import mysql from video.constants import ( access_control, header, product as product_constants, release as release_constants, ) from video.exceptions import InvalidRequest, ProductVideoNotFound from video.logic import approval as approval_logic from video.models.ows import ( product as ows_product, project as ows_project, track as ows_track, ) from video.models.ows.track import get_track_by_id from video.models.sql.classes import ( artist_delivery_settings, carveouts, product_manager_mapping_product, product_video, release, youtube_channel, ) from video.utils import features, status_checker def load_metadata( product_id: int, account_type: int | str | None, account_id: int | str | None ) -> dict[str, Any]: """Load metadata.""" if account_type != access_control.OA: ows_product.check_product_ownership(product_id, account_type, account_id) product_data = product_video.get(product_id) if not product_data: raise ProductVideoNotFound() product_manager_response = product_manager_mapping_product.get_product_manager( product_id ) if product_manager_response: product_data.update( {"product_manager_id": product_manager_response["product_manager_id"]} ) return product_data # Commented out fields are ones we are investigating removing support for. USER_SPECIFIABLE_FIELDS = frozenset( { "latest_pipeline_run_id", "type_of_video", "language_of_video_title", # 'product_manager_id', # 'release_name', "video_title", "version", "product_code", "description", "upc", "isrc", "imprint", "language_of_video_content", "lyrics", "c_line_year", "c_line_copyright_holder", "p_line_year", "p_line_copyright_holder", "new_release", # 'not_for_distribution', "original_release_date", "release_date", "special_instructions", "parental_advisory", "genre_id", "subgenre_id", "keywords", "contributors", "primary_artist_id", "preview_start_time", "thumbnail_path", "thumbnail_at_milliseconds", "custom_thumbnail_path", "channel_selection", "associated_track_id", # 'vendor_release_identifier', } ) def log_unexpected_fields(user_specified_product_metadata: dict[str, Any]) -> None: """Log unexpected fields.""" unexpected_fields = { key: value for key, value in user_specified_product_metadata.items() if key not in USER_SPECIFIABLE_FIELDS } if unexpected_fields: g.log.warning(f"Unexpected product metadata fields: {unexpected_fields}") def save_metadata( product_id: int, data: dict[str, Any], account_type: int | str | None = None, account_id: int | str | None = None, user_id: str | None = None, ) -> dict[str, Any]: """Save metadata.""" log_unexpected_fields(data) ows_product.check_product_ownership( product_id, account_type, account_id, ) with mysql.ar_db_session() as session: if not user_id or not user_id.startswith("oa:"): status_checker.check_release_status_before_action( product_id, frozenset({release_constants.LABEL_PROCESSING}), session=session, ) data, errors = _validate( product_id, data, account_type, account_id, ) if not data: return {"errors": errors} data["release_id"] = product_id if "release_name" in data: data["video_title"] = data["release_name"] del data["release_name"] # This locks the product_video row to prevent race conditions video_data = product_video.upsert(data, session=session) if not video_data: raise InvalidRequest("Product video update failed") if "product_manager_id" in data: product_manager_mapping_product.upsert( data["release_id"], data["product_manager_id"], session=session ) video_data.update({"product_manager_id": data["product_manager_id"]}) release_response = update_release(product_id, data, video_data) release_status = release_response.get("release_status") if ( release_status == release_constants.IN_CONTENT and "releaseDateInPast" in errors ): errors.remove("releaseDateInPast") video_data["errors"] = errors if ( release_status == release_constants.IN_CONTENT and user_id and "oa:" in user_id ): approval_logic.write_to_delivery_tables( product_id, session=session, ) return video_data def bulk_ingest_metadata(data: list[dict[str, Any]]) -> dict[str, Any]: """Bulk ingest metadata. Args: data ([dict]): The list of items to ingest. Returns: dict: Containing the lists of successes and errors. """ successes = [] errors = [] for item in data: success, error = _ingest_item(item) if success: success["item"] = item successes.append(success) if error: error["item"] = item errors.append(error) return {"successes": successes, "errors": errors} def update_upc_for_carveouts(product_id: int, saved_data: dict[str, Any]) -> None: """Check that user is making a upc update.""" release_upc_response = release.get_upc_by_product_id(product_id) if not release_upc_response: return release_upc = release_upc_response.get("upc") if saved_data["upc"] != release_upc: with mysql.ar_db_session() as session: carveouts.update_upc(product_id, saved_data["upc"], session) def update_release( product_id: int, saved_data: dict[str, Any], complete_video_data: dict[str, Any] ) -> dict[str, Any]: """Update release.""" release_data = _map_product_video_data_to_release_data(saved_data) if "keywords" in saved_data and saved_data["keywords"]: release_data["keywords"] = ", ".join(saved_data["keywords"]) if ( ("c_line_year" in saved_data or "c_line_copyright_holder" in saved_data) and complete_video_data["c_line_year"] and complete_video_data["c_line_copyright_holder"] ): release_data["c_line"] = ( str(complete_video_data["c_line_year"]) + " " + complete_video_data["c_line_copyright_holder"] ) if "new_release" in saved_data: release_data["new_release"] = ( release_constants.NEW_RELEASE_NEW if saved_data["new_release"] else release_constants.NEW_RELEASE_CATALOG ) if "upc" in saved_data and saved_data["upc"] and saved_data["upc"] != "": release_data["upc"] = saved_data["upc"] release_data["display_upc"] = saved_data["upc"] if "upc" in saved_data and saved_data["upc"] is None: _make_new_placeholder_upc(product_id, release_data) if "upc" in release_data and release_data["upc"]: update_upc_for_carveouts(product_id, release_data) return release.update(product_id, release_data) def migrate_project( product_id: int, project_id: int, account_type: int | str | None, account_id: int | str | None, ) -> dict[str, Any]: """ Update project_id of a product. Args: product_id (int): Unique Identifier of Product. project_id (int): Unique Identifier of Project. account_type (str): Grass account type i.e vendor or subaccount. account_id (int): Identifier of vendor or subaccount. Returns: dict: product information associated with product_id """ ows_product.check_product_ownership(product_id, account_type, account_id) ows_project_response = ows_project.get_project_by_id(project_id) ows_project.check_project_ownership(project_id, account_type, account_id) artist_id = int(ows_project_response["artist_id"]) release_data = { "project_id": ows_project_response.get("project_id"), "subaccount_id": ows_project_response.get("subaccount_id") or None, "artist_id": artist_id, } product_video.update_primary_artist(product_id, artist_id) return release.update(product_id, release_data) def get_available_channels( product_id: int, account_type: int | str | None, account_id: int | str | None, user_id: str | None, ) -> list[dict[str, Any]]: """Get available channels.""" is_oa_user = user_id and user_id.startswith(header.OA_USER_PREFIX) if not is_oa_user: ows_product.check_product_ownership(product_id, account_type, account_id) product = release.get_by_id_with_vendor(product_id) vendor_id = product["vendor_id"] project_id = product["project_id"] project = ows_project.get_project_by_id(project_id) artist_id = project.get("artist_id") return _get_channels_for_artist(artist_id, account_type, vendor_id) def get_available_channels_by_project_id( project_id: int, account_type: int | str | None, account_id: int | str | None ) -> list[dict[str, Any]]: """Get available channels by project id.""" ows_project.check_project_ownership(project_id, account_type, account_id) project = ows_project.get_project_by_id(project_id) vendor_id = project.get("vendor_id") artist_id = project.get("artist_id") return _get_channels_for_artist(artist_id, account_type, vendor_id) def _get_channels_for_artist( artist_id: int | None, account_type: int | str | None, vendor_id: int | None ) -> list[dict[str, Any]]: """Get the channels for the specified artist.""" all_results: list[dict[str, Any]] = ( artist_delivery_settings.get(artist_id) if artist_id is not None else [] ) if account_type == header.GRASS_ACCOUNT_TYPE_VENDOR or account_type is None: all_results += _add_vevo_controlled(youtube_channel.get(vendor_id)) return _unique_channels(all_results) def _unique_channels(channels: list[dict[str, Any]]) -> list[dict[str, Any]]: """Uniques channels based on name and id tuples while retaining order.""" uniqued_channels = [] seen_name_id_tuples = [] for channel in channels: unique_name_id_tuple = ( channel["channel_name"], channel["channel_id"], ) not in seen_name_id_tuples if unique_name_id_tuple: seen_name_id_tuples.append((channel["channel_name"], channel["channel_id"])) uniqued_channels.append(channel) return uniqued_channels def _add_vevo_controlled(channels: list[dict[str, Any]]) -> list[dict[str, Any]]: for channel in channels: channel["vevo_controlled"] = "No" return channels def _validate( product_id: int | None, data: dict[str, Any], account_type: int | str | None, account_id: int | str | None, is_bulk: bool = False, ) -> tuple[dict[str, Any], list[str]]: """Build up a list of errors.""" errors: list[str] = [] if "isrc" in data and data["isrc"]: _validate_isrc(data, errors, product_id, is_bulk) if ( "product_code" in data and data["product_code"] and account_type and account_id and product_id is not None ): _validate_product_code(data, errors, account_type, account_id, product_id) if "upc" in data and data["upc"]: _validate_upc(data, errors, product_id) if "p_line_year" in data and data["p_line_year"]: _validate_p_line_year(data, errors) if "p_line_copyright_holder" in data and data["p_line_copyright_holder"]: _validate_p_copyright_holder(data, errors) if "c_line_year" in data and data["c_line_year"]: _validate_c_line_year(data, errors) if "c_line_copyright_holder" in data and data["c_line_copyright_holder"]: _validate_c_copyright_holder(data, errors) if not is_bulk and "release_date" in data and data["release_date"]: _validate_release_date(data, errors) if "contributors" in data and data["contributors"]: _validate_contributors(data, errors) return data, errors def _validate_isrc( data: dict[str, Any], errors: list[str], product_id: int | None, is_bulk: bool = False, ) -> None: isrc_pattern = re.compile(r"^[A-Za-z]{2}[0-9A-Za-z]{3}[0-9]{2}[0-9]{5}$") if not isrc_pattern.match(data["isrc"]): data.pop("isrc", None) errors.append("badIsrcFormat") return if not is_bulk: is_isrc_used = ows_track.is_isrc_used(data["isrc"], "music") if is_isrc_used["used"] and is_isrc_used["used_by_product_id"] != product_id: data.pop("isrc", None) errors.append("isrcUsedByMusic") return def _validate_product_code( data: dict[str, Any], errors: list[str], account_type: int | str, account_id: int | str, product_id: int, ) -> None: """Broken in OA, we are okay with that for now.""" product_code_exists_response = ows_product.check_product_code_exists( data["product_code"], account_type, account_id, product_id ) if product_code_exists_response: data.pop("product_code", None) errors.append("productCodeInUse") return def _validate_upc( data: dict[str, Any], errors: list[str], product_id: int | None ) -> None: upc_pattern = re.compile(r"^\d{12,14}$") if not upc_pattern.match(str(data["upc"])): data.pop("upc", None) errors.append("badUpcFormat") return if not is_valid_GTIN(data["upc"]): data.pop("upc", None) errors.append("invalidUpc") return release_by_upc = release.get_by_upc(int(data["upc"])) if release_by_upc and product_id != release_by_upc["id"]: data.pop("upc", None) errors.append("duplicateUpc") return def _validate_p_line_year(data: dict[str, Any], errors: list[str]) -> None: _validate_line_year(data, errors, "p_line_year", "pLineError") def _validate_p_copyright_holder(data: dict[str, Any], errors: list[str]) -> None: _validate_copyright_holder(data, errors, "p_line_copyright_holder", "pLineError") def _validate_c_line_year(data: dict[str, Any], errors: list[str]) -> None: _validate_line_year(data, errors, "c_line_year", "cLineError") def _validate_c_copyright_holder(data: dict[str, Any], errors: list[str]) -> None: _validate_copyright_holder(data, errors, "c_line_copyright_holder", "cLineError") def _validate_line_year( data: dict[str, Any], errors: list[str], data_key: str, error_name: str ) -> None: line_pattern = re.compile(r"^(19|20)\d{2}") if not line_pattern.match(str(data[data_key])): data.pop(data_key, None) errors.append(error_name) return def _validate_copyright_holder( data: dict[str, Any], errors: list[str], data_key: str, error_name: str ) -> None: """Validate the copywriter holder, make sure it doesnt start with a space. to match ows-track validation https://github.com/theorchard/ ows-track/blob/11dead53c0994db654d4367f0995cc78807bb6c8/backend/utils/validation.py#L41 """ line_pattern = re.compile(r"^\S.*$") if not line_pattern.match(str(data[data_key])): data.pop(data_key, None) errors.append(error_name) return def _is_valid_upc(upc: int) -> bool: """Largest valid UPC is 13 digits.""" return upc < 10_000_000_000_000 def _make_new_placeholder_upc(product_id: int, release_data: dict[str, Any]) -> None: """We need to save to releases table on every save to make search work. When a user enters a valid UPC and then switches back to Orchard generated UPC, we need to set back to a placeholder UPC (because we cant set releases upc to null since its the primary key) """ current_release_data = release.get_by_id_with_vendor(product_id) if _is_valid_upc(current_release_data["upc"]): new_placeholder_upc = ows_product.generate_placeholder_upc() release_data["upc"] = new_placeholder_upc release_data["display_upc"] = new_placeholder_upc def _validate_release_date(data: dict[str, Any], errors: list[str]) -> None: """Validate that the release date is in the present or future.""" release_date = datetime.strptime(data["release_date"], "%Y-%m-%d").date() est = ZoneInfo("US/Eastern") if release_date < datetime.now(est).date(): data.pop("release_date", None) errors.append("releaseDateInPast") return def _validate_contributors(data: dict[str, Any], errors: list[str]) -> None: """Validate that contributors have both name and role, and require a Composer.""" contributors = data["contributors"] valid_contributors = [] has_composer = False for contributor in contributors: is_invalid_performer = False if not contributor["role"] and not contributor["name"]: data.pop("contributors") errors.append("contributorNameAndRoleMustBePresent") return if contributor["name"] and not contributor["role"]: data.pop("contributors") errors.append("contributorNameAndRoleMustBePresent") return if contributor["role"] and not contributor["name"]: # We store additional primary artists as 'perfomers' so we make # an exception to the validation to not return an error. if contributor["role"] != "performer": data.pop("contributors") errors.append("contributorNameAndRoleMustBePresent") return else: is_invalid_performer = True if not is_invalid_performer: valid_contributors.append(contributor) # Check if this contributor is a composer (case-insensitive) if contributor.get("role", "").lower() == "composer": has_composer = True if not has_composer and features.is_ccm_vpb_composer_required(): errors.append("composerRequired") data["contributors"] = valid_contributors def _map_product_video_data_to_release_data( product_video_data: dict[str, Any], ) -> dict[str, Any]: """Map some product video data to some release data.""" release_data = {} from_to = { "imprint": "label", "video_title": "release_name", "language_of_video_title": "meta_language", "description": "description", "product_code": "product_code", "language_of_video_content": "language_id", "original_release_date": "original_release_date", "release_date": "release_date", "not_for_distribution": "not_for_distribution", "genre_id": "genre_id", "version": "version", "special_instructions": "special_instructions", "vendor_release_identifier": "vendor_release_identifier", "manufacturer_upc": "manufacturer_upc", "vendor_catalog_number": "vendor_catalog_number", } for product_video_name, release_name in from_to.items(): if product_video_name in product_video_data: release_data[release_name] = product_video_data[product_video_name] if "release_date" in product_video_data: release_data["sale_start_date"] = product_video_data["release_date"] return release_data def _ingest_item( item: dict[str, Any], ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: """Ingest one item as part of a bulk ingest. Args: item (dict): The item to ingest. Returns: tuple: Containing the result and the error. """ vendor_id = item.get("vendor_id") vendor_id = int(vendor_id) if vendor_id else vendor_id subaccount_id = item.get("subaccount_id") subaccount_id = int(subaccount_id) if subaccount_id else subaccount_id account_id = subaccount_id or vendor_id account_type = ( header.GRASS_ACCOUNT_TYPE_SUBACCOUNT if subaccount_id else header.GRASS_ACCOUNT_TYPE_VENDOR ) release_id = item.get("release_id") release_id = int(release_id) if release_id else release_id project_id = item.get("project_id") project_id = int(project_id) if project_id else project_id upc = item.get("upc") primary_artist_id = item.get("primary_artist_id") primary_artist_id = ( int(primary_artist_id) if primary_artist_id else primary_artist_id ) channel_name = item.get("channel_name") vendor_catalog_number = item.get("vendor_catalog_number") item, validation_errors = _validate( release_id, item, account_type, account_id, is_bulk=True, ) if validation_errors: return None, {"type": "ValidationError", "errors": ",".join(validation_errors)} # We first check if the item contains a release_id. # This is the case where we're migrating an old product. # In this case we don't need to create a release since it already exists. if release_id: if not upc: return None, { "type": "ReleaseError", "errors": "If a release_id is provided, a UPC is required.", } if product_video.get(release_id): return None, { "type": "ReleaseError", "errors": "The provided release_id already exists in product_video.", } if not release.get_by_id_with_vendor(release_id): return None, { "type": "ReleaseError", "errors": "The provided release_id does not exist.", } else: # Here we check if the item contains a project_id. # If so, we'll use this project_id to create the release. # If not, we'll first create a project and then create the release. if project_id: try: project_data = ows_project.get_project_by_id(project_id) except HTTPError as e: return None, {"type": "ProjectError", "errors": str(e)} if not _project_belongs_to_account(project_data, account_type, account_id): return None, { "type": "ProjectError", "errors": "The provided project_id does not belong to the requesting account.", } else: project_data = { "vendor_id": vendor_id, "project_name": item.get("project_name"), "project_code": item.get("project_code"), "artist_id": item.get("project_artist_id"), } if subaccount_id: project_data["subaccount_id"] = subaccount_id try: project = ows_project.create_project( project_data, account_type, str(account_id) ) except HTTPError as e: return None, {"type": "ProjectError", "errors": str(e)} project_id = project.get("project_id") # Here we check if the item contains a UPC. # If not, we'll create a placeholder UPC and use it for the release. if not upc: try: upc = ows_product.generate_placeholder_upc() except HTTPError as e: return None, {"type": "ProductError", "errors": str(e)} release_data = { "upc": upc, "vendor_catalog_number": vendor_catalog_number, "release_status": "label_processing", "artist_id": primary_artist_id, "project_id": project_id, "distribution_format_id": ( product_constants.MUSIC_VIDEO_DISTRIBUTION_FORMAT_ID ), "product_type_id": (product_constants.MUSIC_PRODUCT_TYPE_ID), } release_data.update(_map_product_video_data_to_release_data(item)) if subaccount_id: release_data["subaccount_id"] = subaccount_id try: release_response = release.create(release_data) except SQLAlchemyError: return None, {"type": "ReleaseError", "errors": "Failed to create release"} release_id = release_response.get("release_id") # In every cases we end up with a release_id. item["release_id"] = release_id item["upc"] = upc # One more step, we need to find the channel if channel_name: channel_result, channel_error = _find_channel( channel_name, primary_artist_id, account_type, vendor_id ) if channel_error or channel_result is None: return None, channel_error item["channel_selection"] = channel_result["channel_name"] item["vevo_controlled"] = channel_result["vevo_controlled"] # We're finally able to insert a row in product_video. product_video_response = product_video.create( {**item, "migrated_metadata_at": func.now()} ) if not product_video_response: return None, { "type": "ProductVideoError", "errors": "Failed to insert product video", } return {"result": product_video_response}, None def _project_belongs_to_account( project_data: dict[str, Any], account_type: int | str | None, account_id: int | str | None, ) -> bool: """Check if the project belongs to the account.""" if account_type == header.GRASS_ACCOUNT_TYPE_VENDOR: if account_id != project_data.get("vendor_id"): return False else: if account_id != project_data.get("subaccount_id"): return False return True def _find_channel( channel_name: str, artist_id: int | None, account_type: int | str | None, vendor_id: int | None, ) -> tuple[dict[str, Any] | None, dict[str, Any] | None]: """Get the available channels and find the specified one.""" channels = _get_channels_for_artist(artist_id, account_type, vendor_id) if not channels: return None, {"type": "ChannelError", "errors": "No channels available"} for channel in channels: if channel.get("channel_name") == channel_name: return { "channel_name": channel_name, "vevo_controlled": channel.get("vevo_controlled"), }, None return None, {"type": "ChannelError", "errors": "Channel not found"} def get_associated_track( associated_track_id: int | None, ) -> dict[str, Any] | None: """Associated_track property.""" track = get_track_by_id(associated_track_id) return prepare_associated_track_dict(track) def prepare_associated_track_dict( associated_track: dict[str, Any], ) -> dict[str, Any] | None: """Adapts Track to associated track dict.""" if not associated_track: return None associated_track_dict: dict[str, Any] = { "tuid": None, "isrc": None, "track_name": None, } for field in associated_track_dict.keys(): associated_track_dict[field] = associated_track.get(field) return associated_track_dict