# coding: utf-8

# # Exploratory analysis

# Initial exploratory analysis to get a feel of the data. 
# Two files:
# * Streams: 60 Tracks and their stream counts across different categories for specific dates
# * Popularity: Tracks and their popularity for specific dates

# ## Popularity

# In[5]:

get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('pylab', 'inline')
pylab.rcParams['figure.figsize'] = (12, 7)
import pandas as pd
import os
import numpy as np
import matplotlib.pyplot as plt

data_dir = '../data/'


# In[6]:

pop = pd.read_feather(os.path.join(data_dir, 'popularity.feather'))
pop.date = pd.to_datetime(pop.date)


# In[7]:

pop.head()


# In[8]:

number_of_rows = len(pop)
number_of_rows


# In[9]:

number_of_tracks = len(pop.track_id.unique())
number_of_tracks


# In[10]:

# number of popularities per track
pop.groupby('track_id').size().agg([np.min, np.mean, np.median, np.std, np.max])


# In[11]:

date_ranges = pop['date'].apply([np.min, np.max])
date_ranges


# In[12]:

counts_by_date =  pop.groupby('date').size().to_frame('count').reset_index()
counts_by_date.head()


# In[13]:

#plot data
counts_by_date.plot( x = 'date', y = 'count')


# In[14]:

first_date_per_track = pop.groupby('track_id').first()
first_date_per_track.head()


# In[15]:

first_date_per_track.groupby('date').size().plot()


# In[16]:

pop.describe()


# In[17]:

pop.groupby('date').agg([np.mean]).plot()


# In[18]:

aa = pop.groupby('track_id').diff().rename(columns={'date':'date_change', 'day_index':'day_index_change', 'popularity':'popularity_change'})
pop = pd.concat([pop,aa], axis =1)

pop.date_change.fillna(0, inplace = True)
pop['date_change_int'] = pop['date_change'].dt.days
pop['day_index'] = pop.groupby('track_id')['date_change_int'].cumsum()


#pop['day_index_change']=(pop.actual_day_index - pop.day_index)


# In[19]:

#pop['day_index'] = pop.groupby('track_id').cumcount()
pop.head()


# In[20]:

popularity_by_day = pop.groupby('day_index')['popularity'].agg([np.min, np.median, np.mean, np.std, np.max, len])
popularity_by_day.head()


# In[21]:

plt.plot(popularity_by_day.index, popularity_by_day['median'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] + 2*popularity_by_day['std'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] - 2*popularity_by_day['std'])


# In[22]:

plt.plot(popularity_by_day.index, popularity_by_day['median'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] + 2*popularity_by_day['std'])
plt.plot(popularity_by_day.index, popularity_by_day['mean'] - 2*popularity_by_day['std'])
plt.axvline(x = 5)

plt.xlim([-2, 40])


# In[26]:

import matplotlib as mp

def plotAllPops(pop, x_min = 0, x_max = 150, x = 'day_index'):
    cmap = mp.cm.autumn
    by_track_id = pop.groupby('track_id')
    cc = 0
    for name, group in by_track_id:
        plt.plot(group[x], group['popularity'], color=cmap(cc/number_of_tracks), alpha = 0.7)
        cc +=1

    plt.title('Popularity over time')
    plt.xlabel(x)
    plt.xlim(x_min, x_max)
    
plotAllPops(pop)


# In[27]:

plotAllPops(pop, -1, 10)


# In[28]:

plt.plot(popularity_by_day.index, popularity_by_day['std'])
plt.xlim([0,50])
plt.axvline(x = 5)


# In[178]:

plt.plot(popularity_by_day.index, popularity_by_day['len'])


# * How do we know when a track is released? Is it at the first data point? ~Ask Joel~ 
# ### Clean popularity -- remove jumps to 0

# In[29]:

replacment_pop = -1*pop.loc[(pop['popularity'] == 0) & (pop['popularity_change'] < -1), 'popularity_change']

# CLEANING: Remove sharp drops to 0 that transition more than one step  
pop.loc[(pop['popularity'] == 0) & (pop['popularity_change'] < -1), 'popularity'] = replacment_pop
#pop.loc[(pop['popularity'] == 0) & (pop['popularity_change'] < 0)] 

plotAllPops(pop)


# In[31]:

pop['popularity_change'].fillna(0, inplace = True)


# In[34]:

pop_change_by_day = pop.groupby('day_index')['popularity_change'].agg([np.mean, np.std, len]).reset_index()
pop_change_by_day.head()


# In[35]:

plt.plot(pop_change_by_day.day_index, pop_change_by_day['mean'])
plt.axvline(5)
xlim([0,50])


# In[37]:

plt.plot(pop_change_by_day.day_index, pop_change_by_day['std'])
plt.axvline(5)
xlim([0,50])


# ### TODO: Calculate days after release for popularity

# ## Streams

# In[202]:

streams = pd.read_feather('../data/streams_60.feather')


# In[203]:

streams.head()


# In[204]:

len(streams)


# In[205]:

len(streams['track_id'].unique())


# In[206]:

streams['date'].apply([np.min, np.max])


# In[207]:

streams.groupby('region').size()


# In[208]:

unique_tracks = streams['track_id'].unique()
release_dates = pd.read_feather('../data/track_release_dates.feather')
release_dates.derived_release_date = pd.to_datetime(release_dates.derived_release_date)
release_dates.set_index('track_id', inplace = True)
#pop.set_index('track_id', inplace = True)
track_streams_with_pop = streams.join(release_dates, on = 'track_id')
tt = track_streams_with_pop.groupby('track_id').agg({'date':np.min, 'derived_release_date': np.min})
tt['data_on_days_after_release'] = tt['date'] - tt['derived_release_date']
tt.describe()


# In[209]:

tt[tt['data_on_days_after_release'] > pd.Timedelta('0d')]


# In[210]:

tt[tt['data_on_days_after_release'] < pd.Timedelta('-1d')]


# In[211]:

streams[streams['track_id'] == '44mJEP4jnRy7crjl5vtJLk']


# In[212]:

import json
from pprint import pprint

with open('../data/sony_tracks_data.json') as track_data:
    t_d = json.load(track_data)
    
this_stream = [x for x in t_d['results'] if x['track_id'] == '44mJEP4jnRy7crjl5vtJLk']
pprint(this_stream)


# ### Train/ test set
# THIS DOES NOT APPLY RIGHT NOW. --In order to have a minimal test set, if the 60 streams are the full extent of what we can get, set aside 10 tracks as test set. Because of the nature of predictions in time, let's take the last 10 in terms of release date.--
# 
# I'm hopeful this won't be needed, ie, we can use the full sample for testing.

# In[213]:

train_streams = streams


# In[214]:

train_streams.head()


# In[215]:

train_streams.reset_index().to_feather('../data/train_streams.feather')


# ### Training data EDA (Streams)

# In[216]:

train_streams.head()


# In[217]:

train_streams.set_index('track_id', inplace =True)


# In[218]:

train_streams = train_streams.join(release_dates)


# In[219]:

train_streams['days_from_release']=train_streams['date']-train_streams['derived_release_date']
train_streams['all_streams_cumsum'] = train_streams.groupby('track_id')['all_streams'].cumsum()
train_streams['days_from_release'] = (train_streams['days_from_release']/ np.timedelta64(1, 'D')).astype(int)
train_streams.reset_index().to_feather('../data/train_streams_with_release.feather')


# In[220]:

train_streams_100 = train_streams[train_streams['days_from_release'] == 100]
train_streams_100.tail()


# ## Note the below are firstly on 'all_streams' on day 100 - then further down there a similar analysis on all streams cumsum 100

# In[221]:

train_streams_100.reset_index()['all_streams'].apply(np.log).plot.hist(bins = 50)


# In[222]:

pop.set_index('track_id', inplace = True)


# In[223]:

pop_5 = pop[pop['day_index'] == 5]
pop_5 = pop_5[['popularity']]

train_streams_100 = train_streams_100[['all_streams']]
len(train_streams_100)


# In[224]:

df = pop_5.join(train_streams_100,how='inner')
pd.tools.plotting.scatter_matrix(df)


# In[225]:

df['log_all_streams'] = np.log(df['all_streams'])


# In[226]:

pd.tools.plotting.scatter_matrix(df)


# ### This treats all_streams_cumsum

# In[230]:


train_streams_100 = train_streams[train_streams['days_from_release'] == 100]
train_streams_100.reset_index()['all_streams_cumsum'].apply(np.log).plot.hist(bins = 50)


# In[232]:

train_streams_100 = train_streams_100[['all_streams_cumsum']]


# In[233]:

df = pop_5.join(train_streams_100,how='inner')
#pd.tools.plotting.scatter_matrix(df)


# In[234]:

df['log_all_streams_cumsum'] = np.log(df['all_streams_cumsum'])


# In[235]:

pd.tools.plotting.scatter_matrix(df)


# In[ ]:

df.reset_index().to_feather('../data/basemodel_cumsum_outlier_removed.feather')


# In[ ]: