"""Populate epidoses links script. run this in dev with command docker compose run --build episodes-links-importer --env=dev --podcast_id=={podcast_id} --spotify_secret_id={spotify_secret_id} --spotify_client_id={spotify_client_id} """ import argparse import json import re import time from urllib import parse from bs4 import BeautifulSoup import requests from selenium import webdriver from selenium.webdriver.chrome.options import Options from selenium.webdriver.common.by import By import spotipy from spotipy.oauth2 import SpotifyClientCredentials from podcast import config from podcast.constants import stores as podcast_stores from podcast.constants.common import PUBLIC_RSS from podcast.models import episode as episode_model from podcast.models import episode_links as episode_links_model from podcast.models import network as network_model from podcast.models import podcast as podcast_model from podcast.models import podcast_links as podcast_links_model from podcast.utils import api_utils podcast_admin_users = { 'dev': '366', 'qa': '1298', 'prod': '88' } def _get_rss_feed(podcast_slug): return 'https://feeds.megaphone.fm/{}'.format(podcast_slug) def _get_valid_episode_link(response, link): if response.status_code == 200 and response.url != link: return response.url return None def _get_store_names_by_ids(): stores = podcast_links_model.get_stores()['items'] return {store['id']: store['name'] for store in stores} def _get_spotify_podcast_link(podcast_id, spotify_store_id): """Get spotify podcast link for specific podcast.""" current_podcast_links = podcast_links_model.get_podcast_links(podcast_id)['items'] return next((link['link'] for link in current_podcast_links if link['store_id'] == spotify_store_id), None) def _get_store_podcast_links(podcast_id): """Get spotify podcast link for specific podcast.""" current_podcast_links = podcast_links_model.get_podcast_links(podcast_id)['items'] return {link['store_id']: link['link'] for link in current_podcast_links} def _get_spotify_client(): """Get spotify client.""" spotify_creds = SpotifyClientCredentials(config.SPOTIFY_CLIENT_ID, config.SPOTIFY_SECRET_ID) return spotipy.Spotify(client_credentials_manager=spotify_creds) def _get_episodes_from_spotify(link): """Get spotify episodes.""" offset = 0 limit = 50 # fixed upper limit in spotify api spotify_episodes = [] spotify_client = _get_spotify_client() while limit: result = spotify_client.show_episodes( show_id=link, # get episodes using podcast link offset=offset, market='US' ) spotify_episodes += result['items'] if result['next'] is None: limit = 0 offset += limit return spotify_episodes def get_spotify_episode_links( podcast_id, store_id, link, episode, spotify_episodes): """Get spotify episode link by matching episode title.""" for spotify_episode in spotify_episodes: if spotify_episode['name'] == episode['title']: return { 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': spotify_episode['external_urls']['spotify'] } return None def get_spotify_links(episodes, current_episode_ids, link, podcast_id, store_id): """Get new spotify links.""" new_episode_links = [] spotify_episodes = None for episode in episodes: if episode['id'] in current_episode_ids: continue if spotify_episodes is None: spotify_episodes = _get_episodes_from_spotify(link) episode_link = get_spotify_episode_links( podcast_id, store_id, link, episode, spotify_episodes) if episode_link: new_episode_links.append(episode_link) return new_episode_links def _get_apple_podcast_auth_token(apple_podcast_link): """Extract apple podcast authorization token. Use selenium web browser, open apple_podcast_link, scroll to end of page. Get network requestlogs, parse log entry to extract network request details to extract authorization token. """ chrome_options = Options() chrome_options.add_argument('--disable-web-security') chrome_options.add_argument('--disable-site-isolation-trials') chrome_options.add_argument('--allow-running-insecure-content') chrome_options.add_argument('--headless') chrome_options.add_argument('--disable-gpu') chrome_options.add_argument('--no-sandbox') chrome_options.set_capability('acceptInsecureCerts', True) chrome_options.set_capability('goog:loggingPrefs', {'performance': 'ALL'}) selenium_grid_url = 'http://{}:{}/wd/hub'.format( config.GRID_IP_NAME, config.GRID_PORT) driver = webdriver.Remote( command_executor=selenium_grid_url, options=chrome_options ) driver.implicitly_wait(10) # Navigate to the Apple Podcast episodes page driver.get(f'{apple_podcast_link}/episodes') time.sleep(2) # Wait for the page to load scrollable_div = driver.find_element(By.ID, 'scrollable-page') # Scroll to end of page to trigger episodes pagination api driver.execute_script('arguments[0].scrollTop = arguments[0].scrollHeight;', scrollable_div) time.sleep(2) # Allow for smooth scrolling logs = driver.get_log('performance') try: # Parse log entry to extract network request details to get auth token for log_entry in logs: try: log_data = json.loads(log_entry['message'])['message'] if ( log_data['method'] == 'Network.requestWillBeSent' and 'headers' in log_data['params']['request'] ): headers = log_data['params']['request']['headers'] if 'Authorization' in headers: return headers['Authorization'] except Exception: continue return None finally: driver.quit() def _get_apple_episodes(link, apple_id): """Get apple episodes from apple api.""" apple_episodes = [] offset = 0 limit = 300 # max limit for apple episodes api authorization_token = _get_apple_podcast_auth_token(link) if authorization_token: while limit: try: url = '{}/v1/catalog/us/podcasts/{}/episodes?offset={}&limit={}'.format( config.APPLE_API_URL, apple_id, offset, limit) apple_data = requests.get( url, headers={ 'Authorization': authorization_token, 'Origin': config.APPLE_API_ORIGIN } ).json() apple_episodes += apple_data['data'] apple_data['next'] except Exception: limit = 0 else: offset += limit return apple_episodes def get_apple_episode_link(episode, apple_episodes, podcast): """Get apple episode link.""" for apple_episode in apple_episodes: megaphone_uid = episode['megaphone_uid'] attributes = apple_episode.get('attributes', {}) if megaphone_uid and \ '{}.mp3'.format(megaphone_uid) in attributes.get('assetUrl', ''): apple_episode_id = apple_episode.get('id') if apple_episode_id and podcast['feed_type'] == PUBLIC_RSS: episode_model.update_episode(episode['id'], {'apple_id': apple_episode_id}) return attributes['url'] return None def get_apple_links( episodes, current_episode_ids, link, apple_id, podcast, store_id): """Get new apple links.""" new_episode_links = [] apple_episodes = None for episode in episodes: if episode['id'] in current_episode_ids: continue if apple_episodes is None: apple_episodes = _get_apple_episodes(link, apple_id) episode_link = get_apple_episode_link(episode, apple_episodes, podcast) if episode_link: new_episode_links.append({ 'podcast_id': podcast['id'], 'episode_id': episode['id'], 'store_id': store_id, 'link': episode_link }) return new_episode_links def _get_episodes_from_stitcher(podcast_slug): """Get stitcher episodes from stitcher api.""" stitcher_episodes = [] offset = 0 limit = 500 while limit: try: url = '{}/search/show/{}/allEpisodes?count={}&offset={}&sort=publish_date&sort_order=desc'.format( config.STITCHER_API_URL, podcast_slug, limit, offset) stitcher_data = requests.get(url).json() stitcher_episodes += stitcher_data['data']['episodes'] total_count = stitcher_data['orchestration']['total_count'] except Exception: limit = 0 else: offset += limit if total_count <= offset: limit = 0 return stitcher_episodes def get_stitcher_episode_slug(podcast_slug, episode, stitcher_episodes): """Get stitcher episode slug.""" megaphone_uid = episode['megaphone_uid'] for stitcher_episode in stitcher_episodes: if stitcher_episode['title'] == episode['title'].strip() and megaphone_uid \ and megaphone_uid in stitcher_episode['audio_url']: return stitcher_episode['slug'] return None def get_stitcher_links(episodes, current_episode_ids, link, podcast_id, store_id): """Get new stitcher episode links.""" new_episode_links = [] stitcher_episodes = None for episode in episodes: if episode['id'] in current_episode_ids: continue podcast_slug = link.split('/')[-1] if stitcher_episodes is None: stitcher_episodes = _get_episodes_from_stitcher(podcast_slug) episode_slug = get_stitcher_episode_slug(podcast_slug, episode, stitcher_episodes) if episode_slug: new_episode_links.append({ 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': '{}/episode/{}'.format(link, episode_slug) }) return new_episode_links def _scrap_episodes_from_rss_feed(podcast_slug): """Scrap episodes from rss feed.""" episodes = [] rss_feed = _get_rss_feed(podcast_slug) response = requests.get(rss_feed) scraped_rss_feed = BeautifulSoup(response.content, 'xml') for result in scraped_rss_feed.find_all('item'): title = result.find('title').text guid = result.find('guid').text episode_mp3_link = result.find('enclosure').get('url') episodes.append({ 'title': title, 'guid': guid, 'episode_mp3_link': episode_mp3_link }) return episodes def _get_episode_guid(episode, rss_feed_episodes): """Get episode guid.""" megaphone_uid = episode['megaphone_uid'] for rss_feed_episode in rss_feed_episodes: if rss_feed_episode['title'] == episode['title'] and megaphone_uid \ and megaphone_uid in rss_feed_episode['episode_mp3_link']: return rss_feed_episode['guid'] return None def get_radio_public_links(episodes, current_episode_ids, link, podcast_id, store_id, rss_feed_episodes): """Get new radio public links.""" new_episode_links = [] for episode in episodes: if episode['id'] in current_episode_ids: continue guid = _get_episode_guid(episode, rss_feed_episodes) if guid: response = requests.get('{}/{}'.format(link, guid)) episode_link = _get_valid_episode_link(response, link) if episode_link: new_episode_links.append({ 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': episode_link }) return new_episode_links def _get_episodes_from_player_fm(link): """Get player fm episodes from player fm api.""" player_fm_episodes = [] try: url = '{}.json?detail=minimal&episode_detail=full&episode_offset=0&' \ 'episode_order=newest&episode_limit=1000'.format(link) player_fm_data = requests.get(url).json() player_fm_episodes = player_fm_data['episodes'] except Exception: pass return player_fm_episodes def get_player_fm_slug(episode, player_fm_episodes): """Get player fm episode slug.""" megaphone_uid = episode['megaphone_uid'] for player_fm_episode in player_fm_episodes: if player_fm_episode['title'] == episode['title'].strip() and megaphone_uid \ and megaphone_uid in player_fm_episode['url']: return player_fm_episode['slug'] return None def get_player_fm_links(episodes, current_episode_ids, link, podcast_id, store_id): """Get new player fm episode links.""" new_episode_links = [] player_fm_episodes = None for episode in episodes: if episode['id'] in current_episode_ids: continue if player_fm_episodes is None: player_fm_episodes = _get_episodes_from_player_fm(link) episode_slug = get_player_fm_slug(episode, player_fm_episodes) if episode_slug: episode_link = '{}/{}'.format(link, episode_slug) new_episode_links.append({ 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': episode_link }) return new_episode_links def _get_deezer_link(episode, deezer_episodes): for deezer_episode in deezer_episodes: if deezer_episode['title'] == episode['title'].strip(): return 'https://www.deezer.com/episode/{}'.format( deezer_episode['id']) return None def _get_deezer_episodes(deezer_id): offset = 0 total = 1 deezer_episodes = [] while offset < total: url = '{}/podcast/{}/episodes?limit=50&index={}'.format( 'https://api.deezer.com', deezer_id, offset) response = requests.get(url).json() offset += 50 total = response['total'] deezer_episodes = deezer_episodes + response['data'] return deezer_episodes def get_deezer_links(episodes, current_episode_ids, link, podcast_id, store_id): """Get new deezer episode links.""" new_episode_links = [] deezer_episodes = None for episode in episodes: if episode['id'] in current_episode_ids: continue if deezer_episodes is None: deezer_id = link.split('/')[-1] # get podcast ID deezer_episodes = _get_deezer_episodes(deezer_id) episode_link = _get_deezer_link(episode, deezer_episodes) if episode_link: new_episode_links.append({ 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': episode_link }) return new_episode_links def get_podcast_addict_episode_links(link, episode, rss_feed_episodes): """Get podcast addict episode links.""" megaphone_uid = episode['megaphone_uid'] for rss_feed_episode in rss_feed_episodes: if rss_feed_episode['title'] == episode['title'] and megaphone_uid \ and megaphone_uid in rss_feed_episode['episode_mp3_link']: encoded_mp3_link = parse.quote(rss_feed_episode['episode_mp3_link'], safe='') return '{}/episode/{}&podcastId={}'.format( config.PODCAST_ADDICT_URL, encoded_mp3_link, link.split('/')[-1]) return None def get_podcast_addict_links( episodes, current_episode_ids, link, podcast_id, store_id, rss_feed_episodes): """Get new podcast addict links.""" new_episode_links = [] if link.startswith('{}/feed'.format(config.PODCAST_ADDICT_URL)): return new_episode_links for episode in episodes: if episode['id'] in current_episode_ids: continue episode_link = get_podcast_addict_episode_links(link, episode, rss_feed_episodes) if episode_link: new_episode_links.append({ 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': episode_link }) return new_episode_links def _get_driver(): selenium_grid_url = 'http://{}:{}/wd/hub'.format( config.GRID_IP, config.GRID_PORT) driver = webdriver.Remote( command_executor=selenium_grid_url, options=webdriver.ChromeOptions() ) return driver def _get_amazon_link(podcast_title, episode_title, driver, link): titles = '{} {}'.format(podcast_title, episode_title) encoded_title = parse.quote(titles.replace('/', ' '), safe='') url = '{}/search/{}/episodes?filter=IsLibrary%7Cfalse&sc=none'.format( config.AMAZON_URL, encoded_title) try: driver.get(url) driver.implicitly_wait(10) links = driver.find_elements(By.TAG_NAME, 'music-horizontal-item') for episode_link in links: amazon_title = episode_link.get_attribute('primary-text') amazon_link = episode_link.get_attribute('primary-href') amazon_podcast_id = parse.urlparse(amazon_link).path.split('/')[2] link_podcast_id = parse.urlparse(link).path.split('/')[2] if amazon_title == episode_title.strip() and link_podcast_id == amazon_podcast_id: return f'{config.AMAZON_URL}{amazon_link}' except Exception as e: print('-------------------------------------------------------------------------------------------') print('Failed fetching Amazon Link for Podcast {} episode {}'.format(podcast_title, episode_title)) print(e) print('-------------------------------------------------------------------------------------------') return None def get_amazon_links(episodes, current_episode_ids, link, podcast, store_id): """Get new amazon links.""" new_episode_links = [] driver = _get_driver() for episode in episodes: if episode['id'] in current_episode_ids: continue episode_link = _get_amazon_link(podcast['title'], episode['title'], driver, link) if episode_link: new_episode_links.append({ 'podcast_id': podcast['id'], 'episode_id': episode['id'], 'store_id': store_id, 'link': episode_link }) driver.quit() return new_episode_links def _get_pandora_episodes(url, link): """Scrape pandora episodes from script tag using pandora episodes page url.""" response = requests.get(url) if _get_valid_episode_link(response, link): scraped_page = BeautifulSoup(response.content, 'html.parser') substring = 'var storeData =' pattern = re.compile(substring) script = scraped_page.find('script', string=pattern) for var in script.string.split(';\n'): if 'storeData' in var: data = var.strip().replace(substring, '') return json.loads(data)['v1/aesopData/annotateObjects'][0]['annotations'] return None def get_pandora_links(episodes, current_episode_ids, link, podcast_id, store_id): """Get new pandora links.""" new_episode_links = [] pandora_episodes = None try: for episode in episodes: if episode['id'] in current_episode_ids: continue if pandora_episodes is None: pandora_domain_url = f'{config.PANDORA_PODCAST_URL}/podcast' url = f'{pandora_domain_url}/all-episodes{link.split(pandora_domain_url)[1]}' pandora_episodes = _get_pandora_episodes(url, link) for pandora_episode in pandora_episodes.values(): if pandora_episode['name'] == episode['title'].strip(): new_episode_links.append({ 'podcast_id': podcast_id, 'episode_id': episode['id'], 'store_id': store_id, 'link': '{}{}'.format(config.PANDORA_PODCAST_URL, pandora_episode['shareableUrlPath']) }) except Exception as e: print('-------------------------------------------------------------------------------------------') print('Failed fetching Pandora Link for Podcast {} episode {}'.format(podcast_id, episode['id'])) print(e) print('-------------------------------------------------------------------------------------------') return new_episode_links def get_episode_links(store_id, store_name, link, episodes, podcast, rss_feed_episodes): """Get new episode links.""" podcast_id = podcast['id'] apple_id = podcast['apple_id'] slug = podcast['slug'] current_episode_links = episode_links_model.get_episode_links( podcast_id, store_id)['items'] current_episode_ids = [link['episode_id'] for link in current_episode_links] if store_name == podcast_stores.APPLE_PODCAST and apple_id: return get_apple_links( episodes, current_episode_ids, link, apple_id, podcast, store_id) if slug: if store_name == podcast_stores.SPOTIFY: return get_spotify_links( episodes, current_episode_ids, link, podcast_id, store_id) if store_name == podcast_stores.PANDORA: return get_pandora_links( episodes, current_episode_ids, link, podcast_id, store_id) if store_name == podcast_stores.RADIO_PUBLIC and rss_feed_episodes: return get_radio_public_links( episodes, current_episode_ids, link, podcast_id, store_id, rss_feed_episodes) if store_name == podcast_stores.PLAYER_FM: return get_player_fm_links( episodes, current_episode_ids, link, podcast_id, store_id) if store_name == podcast_stores.DEEZER: return get_deezer_links( episodes, current_episode_ids, link, podcast_id, store_id) if store_name == podcast_stores.STITCHER: return get_stitcher_links( episodes, current_episode_ids, link, podcast_id, store_id) if store_name == podcast_stores.PODCAST_ADDICT and rss_feed_episodes: return get_podcast_addict_links( episodes, current_episode_ids, link, podcast_id, store_id, rss_feed_episodes) # TODO: Amazon Music episodes link generation needs to be fixed. # if store_name == podcast_stores.AMAZON_MUSIC: # return get_amazon_links( # episodes, current_episode_ids, link, podcast, store_id) return [] def populate_podcast_episodes_links(podcast, stores): """Populate episode links for specific podcast.""" new_episode_links = [] podcast_id = podcast['id'] rss_feed_episodes = None episodes = episode_model.get_episodes(podcast_id, filter_by_state='DONE')['items'] store_links = _get_store_podcast_links(podcast_id) rss_feed_scraped_stores = [ podcast_stores.RADIO_PUBLIC, podcast_stores.PODCAST_ADDICT ] for store_id, link in store_links.items(): store_name = stores.get(store_id) if rss_feed_episodes is None and store_name in rss_feed_scraped_stores and podcast['slug'] is not None: rss_feed_episodes = _scrap_episodes_from_rss_feed(podcast['slug']) new_links = get_episode_links( store_id, store_name, link, episodes, podcast, rss_feed_episodes) new_episode_links = new_episode_links + new_links if len(new_episode_links): episode_links_model.create_episode_links({ 'links': new_episode_links }) def populate_all_podcasts_episodes_links(): """Fetch all podcasts and populate episode links for each of them.""" networks = network_model.get_networks()['items'] network_ids = [network['id'] for network in networks] stores = _get_store_names_by_ids() podcasts = podcast_model.get_podcasts_by_network_ids(network_ids=network_ids)['items'] for podcast in podcasts: populate_podcast_episodes_links(podcast, stores) def main(): """Extract shell arguments and populate store episodes links.""" argparser = argparse.ArgumentParser(prog='store episodes link importer') argparser.add_argument('--podcast_id', required=False, help='podcast id') argparser.add_argument('--env', required=False, help='env', default='dev') argparser.add_argument('--spotify_secret_id', required=True, help='Spotify Secret ID') argparser.add_argument('--spotify_client_id', required=True, help='Spotify Client ID') args = argparser.parse_args() podcast_id = args.podcast_id env = args.env config.SPOTIFY_CLIENT_ID = args.spotify_client_id config.SPOTIFY_SECRET_ID = args.spotify_secret_id def get_user_id(): """Mock get user id return value.""" return podcast_admin_users[env] api_utils.get_user_id = get_user_id if podcast_id: stores = _get_store_names_by_ids() podcast = podcast_model.get_podcast_by_id(podcast_id) populate_podcast_episodes_links(podcast, stores) else: populate_all_podcasts_episodes_links() if __name__ == '__main__': main()