import pandas as pd import datetime import requests import base64 import json import time from datetime import date from dotenv import load_dotenv load_dotenv() CREDENTIAL_STATUS = False def set_credentials(client_id, client_secret): global CLIENT_ID global CLIENT_SECRET global CREDENTIAL_STATUS CLIENT_ID = client_id CLIENT_SECRET = client_secret CREDENTIAL_STATUS = True def assert_creds(): if not CREDENTIAL_STATUS: raise Exception('Credentials not set! Call set_credentials(client_id, client_secret) first.') status_dict = { 200 : 'OK', 201 : 'Created', 202 : 'Accepted', 204 : 'No Content', 304 : 'Not Modified', 401 : 'Unauthorized', 403 : 'Forbidden', 404 : 'Not Found', 429 : 'Too Many Requests', 500 : 'Internal Server Error', 502 : 'Bad Gateway', 503 : 'Service Unavailable' } def get_token_header(client_id, client_secret): enc = client_id + ':' + client_secret enc = base64.b64encode(enc.encode('ascii')).decode('ascii') return {'Authorization': 'Basic ' + enc} def get_token(sleepytime=5, tries=5): success = False for _ in range(tries): try: response = session.post( 'https://accounts.spotify.com/api/token', headers=get_token_header( client_id=CLIENT_ID, client_secret=CLIENT_SECRET ), data={'grant_type': 'client_credentials'} ) json_response = json.loads(response.text) token = json_response['access_token'] success = True break except: print(f'Could not get token, retrying in {sleepytime} seconds.') time.sleep(sleepytime) if not success: raise Exception(f'Could not get token after {tries} tries.') return token def get_header(token): header = { 'Authorization': 'Bearer ' + token, 'Accept' : 'application/json, text/javascript, */*; q=0.01', 'X-Requested-With': 'XMLHttpRequest', 'Referer': 'https://api.spotify.com', 'Host': 'api.spotify.com' } return header def process_status(response, verbose=False): """Returns 3 booleans: Move on?, Success?, Refresh Token?""" status_code = int(response.status_code) try: status_msg = status_dict[status_code] if verbose: print(f'Request status: {status_msg}') except KeyError: print(f'Uknown status: code {status_code}.') return False, False, False if status_code in [200, 404]: return True, status_code == 200, False elif status_code == 429: retry_after = int(response.headers['retry-after']) print(f'Retrying after {retry_after} seconds.') time.sleep(retry_after) return False, False, True else: # Status code 50X print(f'Retrying after 5 seconds.') time.sleep(5) return False, False, False def start_session(): global session session = requests.Session() session.headers = { 'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36' } def get_response(params, api_url, header, token): # request artists for _ in range(5): time.sleep(0.13) response = session.get(api_url, headers=header, params=params) move_on, success, refresh_token = process_status(response) if refresh_token: token = get_token() if move_on: break if not success: print(f'Failed, skipping') return 'foo', token, False return response, token, True def chunks(lst, n): """Yield successive n-sized chunks from list.""" for i in range(0, len(lst), n): yield lst[i:i + n] def parse_artist_info(artist): try: artist_followers = int(artist['followers']['total']) except KeyError: artist_followers = -1 try: artist_genres = artist['genres'] except KeyError: artist_genres = None try: artist_id = str(artist['id']) except KeyError: artist_id = None try: artist_image_url = str(artist['images'][0]['url']) except KeyError: artist_image_url = None try: artist_name = str(artist['name']) except KeyError: artist_name = None try: artist_popularity = int(artist['popularity']) except KeyError: artist_popularity = -1 try: artist_uri = str(artist['uri']) except KeyError: artist_uri = None return [ date.today(), artist_id, artist_uri, artist_name, artist_image_url, artist_followers, artist_popularity, artist_genres ] def get_artist_info(ids, batchsize=50): assert_creds() start_session() api_url = 'https://api.spotify.com/v1/artists' token = get_token() header = get_header(token) batches = list(chunks(ids, batchsize)) artist_list = [] for b in batches: params = {'ids': ','.join(b)} response, token, success = get_response(params, api_url, header, token) if not success: continue artists = json.loads(response.text)['artists'] for artist in artists: artist_list.append(parse_artist_info(artist)) return pd.DataFrame( artist_list, columns=[ 'date', 'artist_id', 'artist_uri', 'artist_name', 'artist_image', 'n_followers', 'artist_popularity', 'artist_genres' ] ) def parse_playlist_tracks(track, idx): try: added_at = datetime.datetime.strptime(track['added_at'].split('T')[0], '%Y-%m-%d') except KeyError: added_at = None except ValueError: added_at = None try: track_isrc_raw = str(track['track']['external_ids']['isrc']) track_isrc = ''.join(filter(str.isalnum, track_isrc_raw)) # strip the occasional dashes etc. found in isrcs except KeyError: track_isrc = '' try: track_name = str(track['track']['name']) except KeyError: track_name = '' try: album_release_date = datetime.datetime.strptime(track['track']['album']['release_date'], '%Y-%m-%d') except (KeyError, ValueError) as e: album_release_date = None try: album_uri = str(track['track']['album']['uri']) except KeyError: album_uri = '' try: album_n_tracks = int(track['track']['album']['total_tracks']) except KeyError: album_n_tracks = -1 try: album_image_url = str(track['track']['album']['images'][-1]['url']) except KeyError: album_image_url = '' track_artists = [] track_artists_uri = [] try: for artist in track['track']['artists']: track_artists.append(artist['name']) track_artists_uri.append(artist['uri']) try: track_artists_full = ', '.join(track_artists) track_artists = track_artists_full[:1000] track_artists_uri_full = ', '.join(track_artists_uri) track_artists_uri = track_artists_uri_full[:1000] except KeyError: track_artists = '' track_artists_uri = '' except KeyError: track_artists = '' track_artists_uri = '' try: track_duration = int(track['track']['duration_ms']) except KeyError: track_duration = -1 try: track_uri = str(track['track']['uri']) except KeyError: track_uri = '' return [ date.today(), idx, track_uri, track_isrc, track_name, track_duration, track_artists, track_artists_uri, album_uri, album_n_tracks, album_release_date, album_image_url, added_at ] def get_playlist_tracks(id, limit=100, market='US'): assert_creds() start_session() api_url = f'https://api.spotify.com/v1/playlists/{id}/tracks' token = get_token() header = get_header(token) tracks_left = True offset = 0 track_list = [] while tracks_left: params = { 'offset': str(offset), 'limit': str(limit), 'market': str(market) } response, token, success = get_response(params, api_url, header, token) if not success: return playlist = json.loads(response.text)['items'] tracks_left = len(playlist) > 0 idx = offset for track in playlist: track_list.append(parse_playlist_tracks(track, idx)) idx += 1 offset += limit pl_df = pd.DataFrame( track_list, columns=[ 'report_date', 'track_position', 'track_uri', 'track_isrc', 'track_name', 'track_duration', 'track_artists', 'track_artists_uri', 'album_uri', 'album_n_tracks', 'album_release_date', 'album_image_url', 'track_added_at' ] ) pl_df['report_date'] = pd.to_datetime(pl_df['report_date']) pl_df['playlist_uri'] = f'spotify:playlist:{id}' return pl_df def parse_audio_features(track): try: track_id = track['id'] except KeyError: track_id = '' # echonest try: acousticness = track['acousticness'] except KeyError: acousticness = -1 try: danceability = track['danceability'] except KeyError: danceability = -1 try: energy = track['energy'] except KeyError: energy = -1 try: instrumentalness = track['instrumentalness'] except KeyError: instrumentalness = -1 try: liveness = track['liveness'] except KeyError: liveness = -1 try: loudness = track['loudness'] except KeyError: loudness = -1 try: speechiness = track['speechiness'] except KeyError: speechiness = -1 try: valence = track['valence'] except KeyError: valence = -1 # metadata try: duration_ms = int(track['duration_ms']) except KeyError: duration_ms = -1 key_dict = { 0 : 'C', 1 : 'C#/Db', 2 : 'D', 3 : 'D#/Eb', 4 : 'E', 5 : 'F', 6 : 'F#/Gb', 7 : 'G', 8 : 'G#/Ab', 9 : 'A', 10 : 'A#/Bb', 11 : 'B', } mode_dict = { 0 : 'Minor', 1 : 'Major' } try: key = key_dict[track['key']] + '' + mode_dict[track['mode']] except KeyError: key = 'NA' try: tempo = track['tempo'] except KeyError: tempo = -1 try: time_signature = track['time_signature'] except KeyError: time_signature = -1 return [ track_id, acousticness, danceability, energy, instrumentalness, liveness, loudness, speechiness, valence, duration_ms, key, tempo, time_signature, ] def get_audio_features(ids, limit=100): assert_creds() start_session() api_url = f'https://api.spotify.com/v1/audio-features' token = get_token() header = get_header(token) tracks_left = True track_list = [] batch_number = 0 while tracks_left: first = batch_number * limit last = min(batch_number * limit + limit, len(ids)) id_batch = ','.join(ids[first:last]) params = {'ids': id_batch} response, token, success = get_response(params, api_url, header, token) if not success: return tracks = json.loads(response.text)['audio_features'] tracks_left = last < len(ids) for track in tracks: track_list.append(parse_audio_features(track)) batch_number += 1 track_df = pd.DataFrame( track_list, columns=[ 'track_id', 'acousticness', 'danceability', 'energy', 'instrumentalness', 'liveness', 'loudness', 'speechiness', 'valence', 'duration_ms', 'key', 'tempo', 'time_signature', ] ) return track_df