Exploratory analysis¶
Initial exploratory analysis to get a feel of the data. Two files:
- Streams: 60 Tracks and their stream counts across different categories for specific dates
- Popularity: Tracks and their popularity for specific dates
Popularity¶
In [30]:
%matplotlib inline
%pylab inline
pylab.rcParams['figure.figsize'] = (12, 7)
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt
from plotnine import *
pd.options.display.max_columns=500
data_dir = '../data/archive_as_of_friday_20_july/'
In [2]:
pop = pd.read_feather(os.path.join(data_dir, 'popularity.feather'))
pop.date = pd.to_datetime(pop.date)
In [3]:
pop.head()
Out[3]:
In [4]:
number_of_rows = len(pop)
number_of_rows
Out[4]:
In [5]:
number_of_tracks = len(pop.track_id.unique())
number_of_tracks
Out[5]:
In [6]:
# number of popularities per track
pop.groupby('track_id').size().agg([np.min, np.mean, np.median, np.std, np.max])
Out[6]:
In [7]:
date_ranges = pop['date'].apply([np.min, np.max])
date_ranges
Out[7]:
In [8]:
counts_by_date = pop.groupby('date').size().to_frame('count').reset_index()
counts_by_date.head()
Out[8]:
In [9]:
#plot data
counts_by_date.plot( x = 'date', y = 'count')
Out[9]:
In [10]:
first_date_per_track = pop.groupby('track_id').first()
first_date_per_track.head()
Out[10]:
In [11]:
first_date_per_track.groupby('date').size().plot()
Out[11]:
In [12]:
pop.describe()
Out[12]:
In [13]:
pop.groupby('date').agg([np.median]).plot()
Out[13]:
In [14]:
pop.groupby('date').agg([np.mean]).plot()
Out[14]:
In [15]:
aa = pop.groupby('track_id').diff().rename(columns={'date':'date_change', 'day_index':'day_index_change', 'popularity':'popularity_change'})
pop = pd.concat([pop,aa], axis =1)
pop.date_change.fillna(0, inplace = True)
pop['date_change_int'] = pop['date_change'].dt.days
pop['day_index'] = pop.groupby('track_id')['date_change_int'].cumsum()
#pop['day_index_change']=(pop.actual_day_index - pop.day_index)
In [16]:
#pop['day_index'] = pop.groupby('track_id').cumcount()
pop.head()
Out[16]:
In [17]:
popularity_by_day = pop.groupby('day_index')['popularity'].agg([np.min, np.median, np.mean, np.std, np.max, len])
popularity_by_day.head()
Out[17]:
In [18]:
plt.plot(popularity_by_day.index, popularity_by_day['median'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] + 2*popularity_by_day['std'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] - 2*popularity_by_day['std'])
Out[18]:
In [19]:
plt.plot(popularity_by_day.index, popularity_by_day['median'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] + 2*popularity_by_day['std'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] - 2*popularity_by_day['std'])
plt.axvline(x = 5)
plt.xlim([-2, 40])
Out[19]:
In [20]:
import matplotlib as mp
def plotAllPops(pop, x_min = 0, x_max = 150, x = 'day_index'):
cmap = mp.cm.autumn
by_track_id = pop.groupby('track_id')
cc = 0
for name, group in by_track_id:
plt.plot(group[x], group['popularity'], color=cmap(cc/number_of_tracks), alpha = 0.7)
cc +=1
plt.title('Popularity over time')
plt.xlabel(x)
plt.xlim(x_min, x_max)
plotAllPops(pop)
In [21]:
plotAllPops(pop, -1, 10)
In [22]:
plt.plot(popularity_by_day.index, popularity_by_day['std'])
plt.xlim([0,50])
plt.axvline(x = 5)
Out[22]:
In [23]:
plt.plot(popularity_by_day.index, popularity_by_day['len'])
Out[23]:
- How do we know when a track is released? Is it at the first data point? ~Ask Joel~ ### Clean popularity -- remove jumps to 0
In [24]:
replacment_pop = -1*pop.loc[(pop['popularity'] == 0) & (pop['popularity_change'] < -1), 'popularity_change']
# CLEANING: Remove sharp drops to 0 that transition more than one step
pop.loc[(pop['popularity'] == 0) & (pop['popularity_change'] < -1), 'popularity'] = replacment_pop
plotAllPops(pop)
In [25]:
pop['popularity_change'].fillna(0, inplace = True)
In [26]:
pop_change_by_day = pop.groupby('day_index')['popularity_change'].agg([np.mean, np.std, len]).reset_index()
pop_change_by_day.head()
Out[26]:
In [34]:
plt.plot(pop_change_by_day.day_index, pop_change_by_day['mean'])
plt.axvline(5)
xlim([0,50])
Out[34]:
In [37]:
ggplot(pop_change_by_day, aes('day_index', 'mean')) + geom_line() + coord_cartesian(xlim=[0,50])
Out[37]:
In [38]:
plt.plot(pop_change_by_day.day_index, pop_change_by_day['std'])
plt.axvline(5)
xlim([0,50])
Out[38]:
TODO: Calculate days after release for popularity¶
In [39]:
release_dates = pd.read_feather(f'{data_dir}/track_release_dates.feather')
release_dates.derived_release_date = pd.to_datetime(release_dates.derived_release_date)
release_dates.set_index('track_id', inplace = True)
pop = pop.join(release_dates, on = 'track_id', how='left')
In [40]:
pop['days_after_release'] = ((pop['date'] - pop['derived_release_date'])/pd.Timedelta('1 day')).astype(int)
pop.head()
Out[40]:
In [41]:
plotAllPops(pop, 0, 10, 'days_after_release')
In [42]:
plotAllPops(pop, x='days_after_release')
In [43]:
plotAllPops(pop)
- It looks like the using the days after release does not help with the inital days. Maybe the calculations are staggered somehow - didn't I read that somwehre?
- There is a vertical line that is fixed by using the relase date, so it might just edge the previous verison.
In [48]:
#pop.to_feather('../data/archive_as_of_friday_20_july/pop_60_cleaned_with_days_after_release.feather')
pop.drop('date_change', axis = 1).to_feather('../data/archive_as_of_friday_20_july/pop_60_cleaned_with_days_after_release.feather')
Streams¶
This file is from Sony and it's the historical stream counts. The data is only available for artist that were signed.
In [71]:
streams = pd.read_feather(f'{data_dir}/streams_60.feather')
In [72]:
streams.head()
Out[72]:
In [73]:
len(streams)
Out[73]:
In [74]:
len(streams['track_id'].unique())
Out[74]:
In [75]:
streams['date'].apply([np.min, np.max])
Out[75]:
In [76]:
streams.groupby('region').size()
Out[76]:
In [77]:
unique_tracks = streams['track_id'].unique()
#pop.set_index('track_id', inplace = True)
track_streams_with_pop = streams.join(release_dates, on = 'track_id')
tt = track_streams_with_pop.groupby('track_id').agg({'date':np.min, 'derived_release_date': np.min})
tt['data_on_days_after_release'] = tt['date'] - tt['derived_release_date']
tt.describe()
Out[77]:
In [78]:
tt[tt['data_on_days_after_release'] > pd.Timedelta('0d')]
Out[78]:
In [79]:
tt[tt['data_on_days_after_release'] < pd.Timedelta('-1d')]
Out[79]:
In [95]:
#streams[streams['track_id'] == '44mJEP4jnRy7crjl5vtJLk']
In [96]:
import json
from pprint import pprint
with open(f'{data_dir}/sony_tracks_data.json') as track_data:
t_d = json.load(track_data)
this_stream = [x for x in t_d['results'] if x['track_id'] == '44mJEP4jnRy7crjl5vtJLk']
#pprint(this_stream)
Train/ test set¶
THIS DOES NOT APPLY RIGHT NOW. --In order to have a minimal test set, if the 60 streams are the full extent of what we can get, set aside 10 tracks as test set. Because of the nature of predictions in time, let's take the last 10 in terms of release date.--
I'm hopeful this won't be needed, ie, we can use the full sample for testing.
In [83]:
train_streams = streams
In [84]:
train_streams.head()
Out[84]:
In [89]:
train_streams.reset_index().to_feather(f'{data_dir}/train_streams.feather')
Training data EDA (Streams)¶
In [90]:
train_streams.head()
Out[90]:
In [91]:
train_streams.set_index('track_id', inplace =True)
In [92]:
train_streams = train_streams.join(release_dates)
In [94]:
train_streams['days_from_release']=train_streams['date']-train_streams['derived_release_date']
train_streams['all_streams_cumsum'] = train_streams.groupby('track_id')['all_streams'].cumsum()
train_streams['days_from_release'] = (train_streams['days_from_release']/ np.timedelta64(1, 'D')).astype(int)
train_streams.reset_index().to_feather(f'{data_dir}/train_streams_with_release.feather')
Visual EDA for Streams¶
In [129]:
def plotAllStreams(pop, x_min = 0, x_max = 150, x = 'days_from_release', y = 'all_streams_cumsum', log = True):
cmap = mp.cm.autumn
by_track_id = pop.groupby('track_id')
number_of_tracks = len(by_track_id)
cc = 0
for name, group in by_track_id:
y_data = np.log(group[y]) if log else group[y]
plt.plot(group[x], y_data, color=cmap(cc/number_of_tracks), alpha = 0.7)
cc +=1
title = 'Log Stream count over time' if log else 'Stream count over time'
plt.title(title)
plt.xlabel(x)
plt.xlim(x_min, x_max)
In [130]:
plotAllStreams(train_streams)
In [132]:
plotAllStreams(train_streams, x_max = 10, x_min = -2)
In [216]:
plotAllStreams(train_streams, x_max = 40, x_min = -2, y = 'all_streams', log = False)
In [161]:
ts = train_streams.copy().reset_index()
streams_day_1 = ts[ts['days_from_release'] == 0]
pop_5 = pop[pop['days_after_release'] == 5]
streams_day_1_pop5 = pd.merge(streams_day_1, pop_5, on = ['track_id'])
streams_day_1_pop5['log_streams_cumsum'] = np.log(streams_day_1_pop5['all_streams_cumsum'])
ss = streams_day_1_pop5[['log_streams_cumsum', 'popularity']]
pd.tools.plotting.scatter_matrix(ss)
Out[161]:
In [111]:
by_t = train_streams.groupby('track_id')
len(by_t)
Out[111]:
In [97]:
train_streams_100 = train_streams[train_streams['days_from_release'] == 100]
train_streams_100.tail()
Out[97]:
Note the below are firstly on 'all_streams' on day 100 - then further down there a similar analysis on all streams cumsum 100¶
In [98]:
train_streams_100.reset_index()['all_streams'].apply(np.log).plot.hist(bins = 50)
Out[98]:
In [99]:
pop.set_index('track_id', inplace = True)
In [100]:
pop_5 = pop[pop['day_index'] == 5]
pop_5 = pop_5[['popularity']]
train_streams_100 = train_streams_100[['all_streams']]
len(train_streams_100)
Out[100]:
In [101]:
df = pop_5.join(train_streams_100,how='inner')
pd.tools.plotting.scatter_matrix(df)
Out[101]:
In [102]:
df['log_all_streams'] = np.log(df['all_streams'])
In [103]:
pd.tools.plotting.scatter_matrix(df)
Out[103]:
This treats all_streams_cumsum¶
In [104]:
train_streams_100 = train_streams[train_streams['days_from_release'] == 100]
train_streams_100.reset_index()['all_streams_cumsum'].apply(np.log).plot.hist(bins = 50)
Out[104]:
In [105]:
train_streams_100 = train_streams_100[['all_streams_cumsum']]
In [106]:
df = pop_5.join(train_streams_100,how='inner')
#pd.tools.plotting.scatter_matrix(df)
In [107]:
df['log_all_streams_cumsum'] = np.log(df['all_streams_cumsum'])
In [108]:
pd.tools.plotting.scatter_matrix(df)
Out[108]:
In [109]:
df.reset_index().to_feather('../data/basemodel_cumsum_outlier_removed.feather')
Use days after release instead¶
In [113]:
pop_5 = pop[pop['days_after_release'] == 5]
pop_5 = pop_5[['popularity']]
In [114]:
df = pop_5.join(train_streams_100,how='inner')
In [115]:
df['log_all_streams_cumsum'] = np.log(df['all_streams_cumsum'])
In [116]:
pd.tools.plotting.scatter_matrix(df)
Out[116]:
In [117]:
df.reset_index().to_feather('../data/basemodel_cumsum_outlier_removed_days_after_release.feather')
In [118]:
df.head()
Out[118]:
Only use stream dates after pop 5!¶
In [198]:
start_of_stream_count = 5
end_of_stream_count = 100
#train_streams_100 = train_streams[train_streams['days_from_release'] == 100]
#train_streams_100.reset_index()['all_streams_cumsum'].apply(np.log).plot.hist(bins = 50)
pop_5_date = pop_5[['track_id','date']]
streams = pd.read_feather(f'{data_dir}/streams_60.feather')
tts = streams[['track_id', 'date', 'all_streams']]
rel = release_dates.reset_index()
tts = pd.merge(tts, rel, on='track_id')
tts['days_from_release']=tts['date']-tts['derived_release_date']
tts['days_from_release'] = (tts['days_from_release']/ np.timedelta64(1, 'D')).astype(int)
tts = tts[tts['days_from_release'] <= 100]
#tts['all_streams_cumsum'] = tts.groupby('track_id')['all_streams'].cumsum()
#tts.reset_index().to_feather(f'{data_dir}/tts_with_release.feather')
pop_5_date = pop_5_date.rename(columns = {'date':'pop_5_date'})
tts = pd.merge(tts, pop_5_date, on='track_id')
tts['after_pop_5'] = np.where(tts['date'] > tts['pop_5_date'], 'After pop 5', 'Before pop 5')
pp5 = tts.groupby(['track_id','after_pop_5'])[['all_streams']].sum()
pp5s = pp5.unstack()
pp5s.columns = pp5s.columns.droplevel(0)
pd.tools.plotting.scatter_matrix(np.log(pp5s))
Out[198]:
In [202]:
np.corrcoef(pp5s['Before pop 5'], pp5s['After pop 5'])
Out[202]:
In [204]:
from scipy.stats import spearmanr
spearmanr(pp5s['Before pop 5'], pp5s['After pop 5'])
Out[204]:
In [206]:
pp5s['fraction_after'] = pp5s['After pop 5'] / (pp5s['After pop 5'] + pp5s['Before pop 5'])
In [207]:
pp5s.describe()
Out[207]:
In [222]:
import statsmodels.api as sm
model = sm.OLS(np.log(pp5s['After pop 5']), np.log(pp5s['Before pop 5']) )
res = model.fit()
res.summary()
Out[222]:
In [220]:
pp = pd.merge(pp5s.reset_index(), pop_5[['track_id', 'popularity']], on = 'track_id')
pp.head()
Out[220]:
In [224]:
p_model = sm.OLS(np.log(pp['After pop 5']), pp['popularity'])
pres = p_model.fit()
pres.summary()
Out[224]: