"""Backfill ad locations for episodes. run this in dev with command docker compose run --build backfill-ad-locations --env=dev --api_key==megaphone_token --asset_transcoder_api_url=http://localhost:5001 --timestamp_format='%H:%M:%S.%f' """ import argparse import csv import datetime import logging import os import sys import time from flask import g import requests from podcast import config from podcast.api import app from podcast.connectors import mysql from podcast.constants import error from podcast.logic import insertion_point as point_logic from podcast.logic import megaphone as megaphone_logic from podcast.models import episode as episode_model from podcast.models import insertion_point as point_model from podcast.models import ows_asset_transcoder as oat_model from podcast.models import podcast as podcast_model from podcast.utils import api_utils from podcast.utils.exc import OwsError log = logging.getLogger('main') log.setLevel(logging.DEBUG) fmt = logging.Formatter('%(asctime)s - %(levelname)s - %(message)s') sh = logging.StreamHandler(sys.stdout) sh.setFormatter(fmt) log.addHandler(sh) users = { 'dev': '366', 'qa': '1298', 'prod': '88' } request_headers = None AD_TYPE_POST = 'post' ERROR_NOT_POST_ROLL = 'post roll data not found' def convert_timestamp_to_decimals(timestamp, timestamp_format): """Convert timestamp ( HH:MM:SS ) / ( HH:MM:SS.F ) / ( MM:SS.F ) to decimals (25.99). TODO optimize this logic. """ try: [hours, minutes, seconds] = timestamp.split(':') seconds = datetime.timedelta(hours=int(hours), minutes=int(minutes), seconds=int(seconds)).seconds except Exception: time_obj = datetime.datetime.strptime(timestamp, timestamp_format) seconds = datetime.timedelta( hours=time_obj.hour, minutes=time_obj.minute, seconds=time_obj.second, microseconds=time_obj.microsecond ).total_seconds() return seconds def normalize_post_roll_timecode(audio_duration, data, is_autofill_post_roll): """Normalize post roll timecode. Check if post roll timecode doesn't exceed episode audio length. If it does then replace it with episode audio length. """ if audio_duration: audio_duration = audio_duration / 1000 post_roll_data = data[-1] if post_roll_data['point_type'] == AD_TYPE_POST: if post_roll_data['timecode'] > audio_duration or is_autofill_post_roll: post_roll_data['timecode'] = audio_duration else: raise OwsError.not_found(ERROR_NOT_POST_ROLL) return data def get_episode_assets(episode_id, oat_api_url): """Return assets for an episode that are already in assets_final. Args: episode_id (int): The unique identifier of the episode oat_api_url (str): ows-asset-transcoder api url Returns: dict: dict with episode assets """ return oat_model.get_simple_asset_dict(get_assets_from_oat(episode_id, 'episode', oat_api_url)) def get_assets_from_oat(object_id, object_type, oat_api_url): """Return assets for an episode by fetching from oat. Args: episode_id (int): The unique identifier of the episode oat_api_url (str): ows-asset-transcoder api url Returns: dict: dict with episode assets """ global request_headers oat_response = requests.get( '{}/assets-by-ids-and-types'.format(oat_api_url), params={ 'object_ids': [object_id], 'object_types': [object_type] }, headers=request_headers ) if oat_response.status_code > 399: raise OwsError.response_error(oat_response) return oat_response.json()['items'] def get_episode_by_megaphone_id(episode_megaphone_id): """Return an episode by megaphone_id. Args: megaphone_id (str): The episode megaphone_id. Returns: dict: the episode. """ with mysql.pod_db_session(read_only=True) as session: episode = session.query(episode_model.Episode).filter( episode_model.Episode.megaphone_id == episode_megaphone_id, episode_model.Episode.is_deleted.isnot(True) ).first() if not episode: raise OwsError.not_found(error.ERROR_EPISODE_NOT_FOUND) return episode.to_dict() def update_ad_locations_data_in_megaphone( mp_podcast_id, episode_data, assets, network_id): """Update ad location data in megaphone. Args: mp_podcast_id (str): The podcast unique identifier in megaphone. mp_podcast_id (str): The podcast unique identifier in megaphone. assets (list of dict): assets of an episode. network_id (str): The network unique identifier. Returns: dict: the megaphone api response. """ insertion_data = megaphone_logic.create_megaphone_insertion_data( episode_data['insertion_points'], assets['audio_duration'] if 'audio_duration' in assets else None ) megaphone_data = { 'preCount': insertion_data['pre_count'], 'preOffset': insertion_data['pre_offset'], 'postCount': insertion_data['post_count'], 'postOffset': insertion_data['post_offset'], 'expectedAdhash': megaphone_logic.get_ad_hash(episode_data), 'insertionPoints': insertion_data['insertion_points'], 'retainAdLocations': True } megaphone_response = megaphone_logic.call_episode_api( megaphone_data, mp_podcast_id, episode_data['megaphone_id'], network_id, episode_data['id'] ) return megaphone_response def create_ad_locations(podcast, episode_id, data, oat_api_url, is_autofill_post_roll): """Create ad locations / insertion points. Args: podcast_id: The unique identifier of Podcast episode_id: The unique identifier of Episode data: json-payload with insertion points oat_api_url: ows-asset-transcoder api url Returns: response.Response: list of created insertion points. """ print(f'\nCreate ad locations called for episode_id : {episode_id}') global request_headers with mysql.pod_db_session() as session: assets = get_episode_assets(episode_id, oat_api_url) audio_duration = assets.get('audio_duration') normalized_data = normalize_post_roll_timecode(audio_duration, data, is_autofill_post_roll) point_model.create_insertion_points(episode_id, normalized_data, session) update_episode = point_logic.update_ad_inventory_to_match_insertions( podcast['id'], episode_id, normalized_data, session) update_ad_locations_data_in_megaphone( podcast['megaphone_id'], update_episode, assets, podcast['network_id']) print(f'\nSuccessfully created ad locations for episode_id : {episode_id}\n') def backfill_ad_locations(filename, oat_api_url, timestamp_format, is_autofill_post_roll): """Return assets for an episode by fetching from oat. Reads csv file. Gets podcast and episode data from ows-podcast db. Converts timestamp to timecode. Collects all insertion points i.e for all ad types, into list for an episode. Creates insertion points in ows-podcast db and megaphone for each episode at once. Args: filename (int): The filename to access the csv file. oat_api_url (str): ows-asset-transcoder api url """ if not filename: raise Exception('File with name: adLocations.csv not found.') log.info(f'Reading file: {filename}.\n') with open(filename) as csv_file: ep_count = 0 error_count = 0 prev_mp_ep_id = None prev_mp_podcast_id = None data = [] errored_episodes = [] podcast = {} episode = {} reader = csv.DictReader(csv_file) for row in reader: try: mp_episode_id = row['episode_id'] mp_podcast_id = row['podcast_id'] ad_type = row['ad_type'] print(f'Running for megaphone_episode_id: {mp_episode_id} and ad_type: {ad_type}') if prev_mp_podcast_id != mp_podcast_id: podcast = podcast_model.get_podcast_by_megaphone_id( mp_podcast_id) prev_mp_podcast_id = podcast['megaphone_id'] if prev_mp_ep_id != mp_episode_id: data = [] prev_mp_ep_id = mp_episode_id episode = get_episode_by_megaphone_id(mp_episode_id) if mp_episode_id not in errored_episodes: timecode = convert_timestamp_to_decimals(row['timestamp'], timestamp_format) data.append( { 'point_type': row['ad_type'], 'timecode': timecode, 'count': int(row['ad_limit']) } ) if row['ad_type'] == AD_TYPE_POST: create_ad_locations(podcast, episode['id'], data, oat_api_url, is_autofill_post_roll) except Exception as e: data = [] errored_episodes.append(mp_episode_id) error_count += 1 log.info( f'Got an exception for megaphone_episode_id: {mp_episode_id}\n' f'Exception trace: {e}\n' ) ep_count += 1 if ep_count % 10 == 0: print(f'\n sleep time 10 secs and episode count: {ep_count}\n') time.sleep(10) print(f'\nErrored errored count: {error_count} and mp_episodes_id: {errored_episodes} \n') def main(): """Extract shell arguments and start backfilling episode ad locations.""" global request_headers argparser = argparse.ArgumentParser(prog='Backfill ad locations') argparser.add_argument('--api_key', required=True, help='Megaphone API Key') argparser.add_argument('--env', required=True, help='env', default='qa') argparser.add_argument( '--asset_transcoder_api_url', required=True, help='asset_transcoder_api_url, https://qa-ows-asset-transcoder.theorchard.io' ) argparser.add_argument('--timestamp_format', required=True, help='timestamp_format', default='%H:%M:%S.%f') argparser.add_argument('--is_autofill_post_roll', required=False, help='autofill post_roll', default=False) args = argparser.parse_args() oat_api_url = args.asset_transcoder_api_url config.MEGAPHONE_API_TOKEN = args.api_key env = args.env timestamp_format = args.timestamp_format is_autofill_post_roll = args.is_autofill_post_roll filename = os.environ.get('FILENAME') request_headers = { 'Grass-Account-Type': 'vendor', 'Grass-Account-Id': '1', 'Correlation-Id': '12', 'Orchard-Identity-UUID': 'podcast-admin', 'Content-Type': 'application/json' } def get_user_id(): """Mock get user id return value.""" return users[env] api_utils.get_user_id = get_user_id log.info(f'Running ad locations backfill for {env} Environment.\n') with app.app_context(): g.log = log backfill_ad_locations(filename, oat_api_url, timestamp_format, is_autofill_post_roll) if __name__ == '__main__': main()