# coding: utf-8

# # Baseline model variants

# In[43]:

import pandas as pd
import numpy as np
get_ipython().run_line_magic('matplotlib', 'inline')
get_ipython().run_line_magic('pylab', 'inline')
pylab.rcParams['figure.figsize'] = (12, 7)

import matplotlib.pyplot as plt

from sklearn.linear_model import LinearRegression
from sklearn.preprocessing import PolynomialFeatures
from sklearn.pipeline import make_pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.metrics import r2_score
from sklearn.metrics import mean_squared_error


# In[44]:


df = pd.read_feather('../data/basemodel_cumsum.feather')

model = LinearRegression(fit_intercept = True)
x = df[['popularity']]
y = df[['log_all_streams_cumsum']]

def PolynomialRegression(degree=2, **kwargs):
    return make_pipeline(PolynomialFeatures(degree), LinearRegression(**kwargs))

param_grid = {'polynomialfeatures__degree': np.arange(3), 'linearregression__normalize': [True, False]}

grid = GridSearchCV(PolynomialRegression(), param_grid, cv = 5, error_score=mean_squared_error)


# ## Baseline model (2018-05-21 22:12)

# In[45]:

def fitAndEvaluate(x,y,grid):
    grid.fit(x,y)
    model = grid.best_estimator_
    print(f'Best model: {model}')
    y_pred = model.predict(x)

    plt.subplot(1,2,1)
    plt.scatter(y_pred, y_pred - y)
    plt.hlines(y = 0, xmin = min(y_pred), xmax = max(y_pred))
    plt.title('Residual Errors')
    
    plt.subplot(1,2,2)
    plt.scatter(x,y)
    lim = plt.axis()
    plt.scatter(x,y_pred, c = 'red')
    plt.title('Prediction versus actual')

    rmse = np.sqrt(np.sum((y_pred - y)**2))[0]
    print(f'RMSE: {rmse}')
    print(f'Rsq: {r2_score(y,y_pred)}')
    

fitAndEvaluate(x,y,grid)


# ## Baseline with outlier removed (2018-05-22 22:15)
# * Evaluate outlier at 0 popularity. It seems to skew the results. It looks like linear would be a good fit here actually

# In[46]:

df.sort_values('popularity').head()


# ### Popularity 0 outlier
# There is an outlier that has massive impact on the curve with 0 popularity, ,this could be some corner case but also some data error. Look into this explicitly.
# * 0A488iaPeDAUP5q7Jm3paF
# 
# **UPDATE this was caused by an erratic jump to 0 and back**
# 
# 

# In[47]:

df = pd.read_feather('../data/basemodel_cumsum_outlier_removed.feather')


# In[48]:

x = df[['popularity']]
y = df[['log_all_streams_cumsum']]


# In[49]:

fitAndEvaluate(x,y,grid)


# In[50]:

import statsmodel.stats


# In[ ]: