"""Examine YouTube Videos.""" import csv import os import sys import time import dotenv import isodate # type: ignore import requests from connectors.database import Client as DatabaseClient from connectors.youtube import Client as YouTubeClient dotenv.load_dotenv() # initialize clients and verify auth yt_client = YouTubeClient() db_client = DatabaseClient( os.environ["DB_HOST"], os.environ["DB_USER"], os.environ["DB_PASS"], os.environ["DB_SCHEMA"], int(os.environ.get("DB_PORT", 3306)), ) class Complete(Exception): pass class InConflict(Exception): pass class VideoTooShort(Exception): pass class ReferenceStatusUnexpected(Exception): pass class ReferenceInsertError(Exception): pass class NeedsManualProcessing(Exception): pass class BlockedByYouTube(Exception): pass class BaconUnprocessable(Exception): pass class NeedsAttention(Exception): pass class UnableToTriage(Exception): pass def main(rows: list[dict]) -> None: db_rows = db_client.get_videos([x["youtube_video_id"] for x in rows]) for row in rows: row.update(db_rows.get(row["youtube_video_id"], {})) try: process(row) except ( InConflict, NeedsManualProcessing, NeedsAttention, VideoTooShort, Complete, BlockedByYouTube, BaconUnprocessable, ReferenceStatusUnexpected, ReferenceInsertError, UnableToTriage, ) as e: row["notes"] = str(e) row["status"] = type(e).__name__ def process(row: dict) -> None: print(row["youtube_video_id"]) # check channel setup in our database if row["monetization_status"] and row["monetization_status"] != "monetizing": raise BaconUnprocessable("Channel monetization status is not monetizing") if row["auto_claim"] and row["auto_claim"] != "yes": raise BaconUnprocessable("Channel auto claim is not set to yes") # verify video exists in YouTube videos = yt_client.get_videos([row["youtube_video_id"]])["items"] if not videos: raise BaconUnprocessable("Video not found on YouTube") video = videos[0] # processing failed and needs to be reset and retried if row["in_processing"] == "Y": raise NeedsManualProcessing( f"UPDATE youtube_channel_video_status SET in_processing = 'N' WHERE youtube_video_id = '{row['youtube_video_id']}'; -- then rerun videoIds in bulk claiming tool and then this script again" ) # data looks good, preform more verification checks elif row["in_processing"] == "N": if not row["youtube_claim_id"]: raise NeedsManualProcessing( "Re-run using Bulk Claiming Tool in OA and try this script again" ) elif not row["youtube_reference_id"]: _process_missing_reference(row) else: _process_verify(row) # video not in database else: if video["contentDetails"]["licensedContent"]: raise BaconUnprocessable("Video is already claimed outside of our system.") video_channel_id = video["snippet"]["channelId"] db_channel = db_client.get_channel(video_channel_id) if not db_channel: raise BaconUnprocessable(f"Channel {video_channel_id} not in database") elif db_channel["auto_claim"] != "yes": raise BaconUnprocessable( f"Channel {video_channel_id} auto claim is not set to yes" ) elif db_channel["monetization_status"] != "monetizing": raise BaconUnprocessable( f"Channel {video_channel_id} monetization status is {db_channel['monetization_status']}" ) raise NeedsManualProcessing( "Video not in database - try manual bulk claiming tool" ) raise UnableToTriage("Row in unexpected case for review") def _process_verify(row: dict) -> None: # verify claim claim = yt_client.get_claim(row["youtube_claim_id"], row["cmsa_content_owner"]) if claim["status"] != "active": raise NeedsAttention( f"Claim is not active - status: {claim['status']} ; ask reporter if claim should be reactivated or new one created" ) if claim["videoId"] != row["youtube_video_id"]: raise NeedsAttention(f"Claim videoId does not match video - {claim['videoId']}") # verify reference reference = yt_client.get_reference( row["youtube_reference_id"], row["cmsa_content_owner"] ) if reference["status"] != "active": if reference["status"] == "deleted": if reference["statusReason"] == "PROCESSING_FAILED": raise BlockedByYouTube("Reference status: deleted - PROCESSING_FAILED") elif reference["statusReason"] == "INSUFFICIENT_LENGTH": raise VideoTooShort("Reference status: deleted - INSUFFICIENT_LENGTH") if reference["status"] == "inactive": status_reason = reference.get("statusReason") if status_reason == "CLAIM_DISABLED_FOR_MATCHING": raise NeedsAttention( "Reference status: inactive - CLAIM_DISABLED_FOR_MATCHING" ) elif status_reason == "DUPLICATE_FOR_OWNERS": raise NeedsAttention( f"Reference status: inactive - DUPLICATE_FOR_OWNERS ({reference['duplicateLeader']})" ) else: _process_missing_reference(row) raise ReferenceStatusUnexpected( f"Reference is not active - status: {reference['status']} - {reference.get('statusReason', 'unknown')}" ) if reference["assetId"] != row["youtube_asset_id"]: asset = yt_client.get_asset(row["youtube_asset_id"], row["cmsa_content_owner"]) if row["youtube_asset_id"] in asset.get("aliasId", []): row["youtube_asset_id"] += f" alias to {asset['id']}" else: raise NeedsAttention( f"Reference assetId does not match asset - {reference['assetId']}" ) if reference["videoId"] != row["youtube_video_id"]: raise NeedsAttention( f"Reference videoId does not match video - {reference['videoId']}" ) raise Complete("Everything looks good - no action needed") def _process_missing_reference(row: dict) -> None: try: videos = yt_client.get_videos([row["youtube_video_id"]])["items"] if not videos: raise BaconUnprocessable("Video not found on YouTube") video = videos[0] duration_str = video["contentDetails"]["duration"] # https://github.com/theorchard/orchard/blob/master/src/Service/Api/Youtube/Response/VideosListTrackQueueWriter.php#L105 duration_seconds = isodate.parse_duration(duration_str).seconds if duration_seconds < 20: raise VideoTooShort(f"Video duration {duration_seconds} < 20 seconds") # attempt to create reference new_reference = yt_client.insert_reference( row["youtube_claim_id"], row["cmsa_content_owner"] ) # wait for new reference to be checked while True: new_ref = yt_client.get_reference( new_reference["id"], row["cmsa_content_owner"] ) if new_ref["status"] == "active": active_reference = new_ref elif new_ref["status"] == "inactive": active_references = [ x for x in yt_client.get_references( new_reference["assetId"], row["cmsa_content_owner"] )["items"] if x["status"] == "active" ] # verify expectations if len(active_references) == 0: raise ReferenceInsertError( f"No active references for asset {new_reference['assetId']}" ) if len(active_references) > 1: raise ReferenceInsertError( f"Too many active references for asset {new_reference['assetId']}" ) active_reference = active_references[0] # detect merged assets if active_reference["videoId"] != row["youtube_video_id"]: asset = yt_client.get_asset( active_reference["assetId"], row["cmsa_content_owner"] ) if row["youtube_asset_id"] in asset["aliasId"]: raise ReferenceInsertError( f"Asset was merged with {active_reference['assetId']} - {active_reference['videoId']}" ) raise ReferenceInsertError( f"Reference {active_reference['id']} videoId does not match - {active_reference['videoId']}" ) elif new_ref["status"] == "deleted": raise BlockedByYouTube( f"New reference ({new_ref['id']}) status: deleted - {new_ref.get('statusReason', 'unknown')}" ) else: time.sleep(10) continue # output sql to repair raise NeedsManualProcessing( f"UPDATE youtube_channel_video_status SET youtube_reference_id = '{active_reference['id']}', youtube_asset_id = '{active_reference['assetId']}' WHERE youtube_video_id = '{active_reference['videoId']}';" ) except requests.exceptions.HTTPError as e: if e.response.status_code == 400: error = e.response.json() message = error["error"]["message"] # expected error, try to find reference's owner if message.startswith( "A reference for the same content has already been created" ): reference_id = message.split(" ")[-1][:-1] reference = yt_client.get_reference( reference_id, row["cmsa_content_owner"] ) reference_owner_id = reference["origination"]["owner"] owner = yt_client.get_owner( reference_owner_id, row["cmsa_content_owner"] ) raise InConflict( f"reference ({reference_id}) asset ({reference['assetId']}) status ({reference['status']}) - {owner['displayName']} {owner['id']} {owner['conflictNotificationEmail']}" ) elif message == "Invalid Value": raise BlockedByYouTube( "HTTP 400 - Invalid Value when trying to create reference" ) print(e.response.json()) raise e if __name__ == "__main__": # check and load CLI parametets if len(sys.argv) < 3: print("Usage: python run.py ") exit(1) video_ids = set(sys.argv[1].split(",")) output_filename = sys.argv[2] # output data format columns = { "last_updated": "", "youtube_video_id": "", "status": "backlog", "notes": "", "cmsa_content_owner": "", "monetization_status": "", "auto_claim": "", "in_processing": "", "youtube_channel_id": "", "youtube_asset_id": "", "youtube_claim_id": "", "youtube_reference_id": "", "error_type": "", } # fill out empty rows with column IDs rows = [columns.copy() | {"youtube_video_id": video_id} for video_id in video_ids] # run program try: main(rows) finally: with open(output_filename, "w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=columns.keys()) writer.writeheader() writer.writerows(rows)