"""Sound Recordings Takedown - YouTube Content ID.""" import asyncio from datetime import datetime, timedelta import json import os import sys from typing import List, Tuple import config import pandas as pd import sentry_sdk from src.connectors import datadog from src.models import ( ows_masters_registry, ows_vectororder, script_status, snowflake as snowflake_model, ) from src.utils import constants, ses, utils logger = config.setup_logger(__name__) def send_upcs_to_registry(upcs: list[str]) -> dict[str, str]: """Send a list of UPCs to the masters registry for ownership updates, handling batching and errors.""" if not upcs: logger.info('No UPCs to send to Masters Registry.') return {} results, skipped_upcs = ows_masters_registry.update_registry(upcs) failed = [(status, text) for status, text in results if status == 'ERROR'] if failed: raise RuntimeError(f'Masters Registry update failed for {len(failed)}/{len(results)} batch(es): {failed}') return skipped_upcs def populate_release_eligibility(takedown_df: pd.DataFrame) -> pd.DataFrame: """Check eligibility for all UPCs and populate Release Eligibility and Release Update columns.""" upcs = takedown_df[constants.RELEASE_UPC].drop_duplicates().astype(str).tolist() eligibility_map = build_eligibility_map(upcs) takedown_df['Release Eligibility'] = takedown_df[constants.RELEASE_UPC].astype(str).map(eligibility_map).fillna('') takedown_df['Release Update'] = 'Y' logger.info('Release Eligibility and Release Update columns populated.') return takedown_df def update_release_registry(takedown_df: pd.DataFrame) -> pd.DataFrame: """Send UPCs where Release Update is Y to the masters registry.""" try: registry_upcs = takedown_df[constants.RELEASE_UPC].dropna().drop_duplicates().astype(str).tolist() logger.info(f'Sending {len(registry_upcs)} unique UPCs from the takedown data to Masters Registry') skipped = send_upcs_to_registry(registry_upcs) if skipped: takedown_df['Registry Skip Reason'] = takedown_df[constants.RELEASE_UPC].map(skipped).fillna('') return takedown_df except Exception as e: logger.exception(f'Failed to update Masters Registry via release data: {e}') raise def update_isrc_registry(isrc_track_df: pd.DataFrame) -> pd.DataFrame: """Send deduplicated UPCs from the ISRC lookup results to the masters registry.""" try: isrc_lookup_upcs = isrc_track_df['Display UPC'].dropna().drop_duplicates().astype(str).tolist() logger.info(f'Sending {len(isrc_lookup_upcs)} unique UPCs from ISRC lookup to Masters Registry') skipped = send_upcs_to_registry(isrc_lookup_upcs) if skipped: isrc_track_df['Registry Skip Reason'] = isrc_track_df['Display UPC'].map(skipped).fillna('') return isrc_track_df except Exception as e: logger.exception(f'Failed to update Masters Registry via ISRC lookup: {e}') raise async def run_all( upcs: List[str], retries: int = 3, backoff_factor: float = 0.5, ) -> List[Tuple[str, str, str]]: """ Run eligibility checks for all UPCs concurrently. Args: upcs (List[str]): List of UPC codes. retries (int): Number of retry attempts. backoff_factor (float): Base wait time (doubles each retry). """ results = [] semaphore = asyncio.Semaphore(config.CONCURRENCY) async def bounded_check(upc: str) -> Tuple[str, str, str]: async with semaphore: return await ows_vectororder.check_product_eligibility(upc, retries, backoff_factor) tasks = [bounded_check(upc) for upc in upcs] for coro in asyncio.as_completed(tasks): upc, status, text = await coro results.append((upc, status, text)) logger.info(f'[UPC {upc}] {status}: {text}') return results def build_eligibility_map(upcs: List[str]) -> dict: """Run eligibility checks for all UPCs and return a dict mapping UPC → is_eligible string.""" if not upcs: logger.info('No UPCs provided for eligibility check.') return {} results = asyncio.run(run_all(upcs)) result = {} for upc, status, text in results: if status == 'ERROR': result[upc] = 'ERROR' continue try: resp_json = json.loads(text) result[upc] = str(resp_json.get('is_eligible') or False).upper() except Exception: result[upc] = 'ERROR' return result if __name__ == '__main__': """Run the sound recordings takedown process.""" try: sentry_dsn = config.SENTRY_DSN sentry_sdk.init(sentry_dsn) now = datetime.now() timestamp = now.strftime('%Y%m%d_%H%M%S') current_run = now.strftime('%Y-%m-%d') last_run = (now - timedelta(days=1)).strftime('%Y-%m-%d') logger.info(f'Fetching takedown data for {last_run}.') takedown_df = snowflake_model.fetch_raw_data(last_run, current_run) if takedown_df[constants.RELEASE_UPC].dropna().empty: logger.info('No Release UPCs found. Exiting.') datadog.publish_metric('sound_recordings_takedown.success', 1) sys.exit(0) takedown_df = populate_release_eligibility(takedown_df) isrc_track_df = snowflake_model.fetch_isrc_track_data(takedown_df) if config.SEND_UPCS_TO_MASTERS_REGISTRY: takedown_df = update_release_registry(takedown_df) isrc_track_df = update_isrc_registry(isrc_track_df) else: logger.info('Skipping Masters Registry updates based on configuration.') output_file = os.path.join(constants.LOCAL_DIR, f'{constants.OUTPUT_FILENAME_PREFIX}_{timestamp}.xlsx') utils.export_and_upload(takedown_df, isrc_track_df, output_file) script_status.update_last_run(now) utils.send_success_email(output_file, os.path.basename(output_file)) datadog.publish_metric('sound_recordings_takedown.success', 1) logger.info('Script completed.') except Exception as e: logger.exception(f'Error in sound-recordings-takedown script: {str(e)}') datadog.publish_metric('sound_recordings_takedown.error', 1) subject = ( constants.FAILURE_EMAIL_SUBJECT if config.ENVIRONMENT == config.PROD_ENVIRONMENT else f'{constants.FAILURE_EMAIL_SUBJECT} ({config.ENVIRONMENT.upper()})' ) message = f'Script failed with exception:\n{str(e)}' ses.send_email( recipients=config.EMAIL_RECIPIENTS, sender=config.EMAIL_SENDER, subject=subject, message=message ) raise