from db.art_relations import conn as art_relations_conn from youtube import api from youtube import config import boto3 import copy from datetime import date, timedelta import json from retrying import retry import sys import time """Back BACON This script crawls the YouTube API to find any videos that BACON has failed to claim and/or fingerprint in the past X number of days. Here's what it does: - Gets all BACON-enabled auto-claiming channels from the DB - Gets all new videos that were uploaded to each channel - Fetches any claims that have been made on those videos - Fetches references for any claims - Spits out the following, broken down by CMS account: - Unclaimed videos - Claimed videos that are missing references Dependencies: - art_relations credentials (see art_relations.py) - YT service account key file (stored under youtube/keys, configured in youtube/config.py) Make sure the service account has been authorized for the necessary CMS accounts. Otherwise, you will get errors and you will cry. Enjoy. """ email_message = '' @retry(stop_max_attempt_number=10, wait_fixed=0.5) def execute_call(youtube_call): return youtube_call.execute() def filter_unclaimed_videos(content_owner, video_ids): vid_ids_string = ','.join(video_ids) print('Fetching claims for ' + content_owner) partner = config.api_mapping.youtube_partner list_call = partner.claimSearch().list( videoId=vid_ids_string, onBehalfOfContentOwner=content_owner) list_results = execute_call(list_call) items = list_results.get('items') or [] claimed_video_ids = [] missing_reference = [] for item in items: video_id = item.get('videoId') claimed_video_ids.append(video_id) # see if there are any references asset_id = item.get('assetId') references = list_references(asset_id, content_owner) if not references: missing_reference.append({'videoId': video_id, 'assetId': asset_id}) unclaimed_ids = set(video_ids) - set(claimed_video_ids) return unclaimed_ids, missing_reference def list_references(asset_id, content_owner): partner = config.api_mapping.youtube_partner list_call = partner.references().list( assetId=asset_id, onBehalfOfContentOwner=content_owner) list_results = execute_call(list_call) items = list_results.get('items') or [] return items def fetch_uploads(channel_id, published_after): video_ids = [] data = config.api_mapping.youtube_data keep_going = True token = None while keep_going: print('Fetching activities for channel ' + channel_id) list_call = data.activities().list( channelId=channel_id, publishedAfter=published_after, maxResults=50, pageToken=token, part='id,snippet,contentDetails') list_results = execute_call(list_call) activities = list_results.get('items') or [] for activity in activities: details = activity.get('contentDetails') if details and 'upload' in details: video_ids.append(details.get('upload').get('videoId')) token = list_results.get('nextPageToken') if not token: keep_going = False time.sleep(0.0004) return video_ids def get_claims(video_id, owner): partner = config.api_mapping.youtube_partner search_call = partner.claimSearch().list( videoId=video_id, onBehalfOfContentOwner=owner, includeThirdPartyClaims=True) result = execute_call(search_call) items = result.get('items') or [] return items def insert_reference(claim_id, owner, content_type): partner = config.api_mapping.youtube_partner insert_call = partner.references().insert( claimId=claim_id, onBehalfOfContentOwner=owner, body={'contentType': content_type}) return execute_call(insert_call) def add_missing_reference(owner, video_id): try: print('Processing video {}...'.format(video_id)) claims = get_claims(video_id, owner) if len(claims) != 1: print('Got {} claims'.format(len(claims))) return claim = claims[0] if claim.get('thirdPartyClaim'): print('Got a third-party claim') return asset_id = claim.get('assetId') if not asset_id: print('Got no asset ID') return claim_id = claim.get('id') if not claim_id: print('Got no claim ID') return content_type = claim.get('contentType') if not content_type: print('Got no content type') return existing_refs = list_references(asset_id, owner) if existing_refs: print('We already have references for asset {}'.format(asset_id)) print(existing_refs) return reference = insert_reference(claim_id, owner, content_type) print('Created reference for video {}, asset {}, claim {}'.format(video_id, asset_id, claim_id)) add_to_email('Created reference for video {}, asset {}, claim {}'.format(video_id, asset_id, claim_id)) add_to_email('\n') print(reference) return except Exception as e: print('BARF on video {}'.format(video_id)) print(str(e)) def get_last_bacon_claim_date(): last_claim_query = """select MAX(claim_insert_time) as max_claim from youtube_channel_video_status""" cursor = art_relations_conn.cursor() cursor.execute(last_claim_query) result = cursor.fetchone() return result['max_claim'].strftime('%Y-%m-%d %H:%M') def add_to_email(message): global email_message email_message += message if len(sys.argv) < 2: print('Usage: {} num_days'.format(sys.argv[0])) exit() days_ago = int(sys.argv[1]) print('Looking for activities starting {} day(s) ago'.format(days_ago)) channels_query = """SELECT ytc.id, ytc.youtube_channel_id, cmsa.cmsa_content_owner, ytcat.youtube_channel_asset_type FROM youtube_channel ytc JOIN youtube_channel_cms_account_history hist ON ytc.youtube_channel_cms_account_history_id = hist.id JOIN youtube_channel_cms_account cmsa ON hist.youtube_channel_cms_account_id = cmsa.id JOIN youtube_channel_asset_types ytcat ON ytc.youtube_channel_asset_type_id = ytcat.id WHERE ytc.auto_claim = 'yes'""" cursor = art_relations_conn.cursor() cursor.execute(channels_query) channels = [] for item in cursor.fetchall(): channels.append({ 'orchardChannelId': int(item['id']), 'youtubeChannelId': item['youtube_channel_id'], 'contentOwner': item['cmsa_content_owner'], 'assetType': item['youtube_channel_asset_type']}) start_date = date.today() - timedelta(days=days_ago) pub_after = str(start_date) + 'T00:00:00.00-05:00' print('Grabbing all videos published after {}'.format(pub_after)) raw_video_ids = {} for line in channels: owner = line.get('contentOwner') if owner not in raw_video_ids: raw_video_ids[owner] = [] raw_video_ids[owner] += fetch_uploads( line.get('youtubeChannelId'), pub_after) vids_per_search = 300 unclaimed_video_ids = {} all_missing_reference = {} for owner in raw_video_ids.keys(): video_ids_for_owner = raw_video_ids[owner] print('RAW video count for ' + owner + ':') print(len(video_ids_for_owner)) filtered_video_ids = [] missing_reference = [] offset = 0 while offset < len(video_ids_for_owner): cutoff = offset + vids_per_search id_slice = video_ids_for_owner[offset:cutoff] more_unclaimed_ids, more_missing_reference = filter_unclaimed_videos( owner, id_slice) filtered_video_ids += more_unclaimed_ids missing_reference += more_missing_reference time.sleep(0.05) offset += vids_per_search unclaimed_video_ids[owner] = filtered_video_ids all_missing_reference[owner] = missing_reference print('Unclaimed Video IDs') add_to_email('Unclaimed Video IDs') add_to_email('\n\n') for owner in unclaimed_video_ids.keys(): print("Content Owner: {}".format(owner)) add_to_email("Content Owner: {}".format(owner)) add_to_email('\n') print(','.join(unclaimed_video_ids[owner])) add_to_email(','.join(unclaimed_video_ids[owner])) add_to_email('\n\n') print('Claimed Videos Missing References') for owner in all_missing_reference.keys(): print("Content Owner: {}".format(owner)) for asset_data in all_missing_reference[owner]: print("Video ID: {}, Asset ID: {}".format( asset_data['videoId'], asset_data['assetId'])) add_missing_reference(owner,asset_data['videoId']) add_to_email('\n') last_bacon_claim_date = get_last_bacon_claim_date() add_to_email('Last BACON Claim: ' + last_bacon_claim_date + '\n') add_to_email('(Note that this may be delayed due to server replication)' + '\n') sns_client = boto3.client('sns') response = sns_client.publish( TopicArn=config.BACON_STATUS_ARN, Subject='BACON Status', Message=email_message )