What impact does the first 5 days have on the prediction

In [9]:
import pandas as pd
import numpy as np

from plotnine import *

import statsmodels.api as sm
from sklearn.model_selection import train_test_split
from statsmodels.sandbox.regression.predstd import wls_prediction_std

from plotnine import *
import plotnine.options
plotnine.options.figure_size = (16,8)

import warnings
warnings.simplefilter(action='ignore', category=FutureWarning)

pd.set_option('display.max_columns', None)
In [10]:
sp = pd.read_feather('../data/archive_as_of_friday_20_july/streams_post_day_5_pop.feather')
sp.head()
Out[10]:
track_id streams_cumsum_5 pop_5 all_streams_cumsum cum_sum_after_pop_5 log_cum_sum_after_pop_5 log_cum_sum_5 log_all_streams_cumsum fraction_of_stream_pre_5
0 00kzys67XYXiB31cSP5jfo 58838 40.0 473673 414835 12.935636 10.982543 13.068272 0.110492
1 00mc2RHScEYMEFlc7FRGaK 4976 26.0 93919 88943 11.395751 8.512382 11.450188 0.050316
2 04HzRAn3BJaIvmhpvc1GVT 190131 51.0 1426517 1236386 14.027703 12.155469 14.170746 0.117608
3 04qrVtScdD4IBGSL5q6yEv 131276 47.0 2637515 2506239 14.734294 11.785057 14.785348 0.047413
4 09aaq7feVx9Jykdw0f00QU 168800 50.0 1371458 1202658 14.000045 12.036470 14.131385 0.109592
In [13]:
ggplot(sp, aes('pop_5', 'log_cum_sum_after_pop_5')) + geom_point() + geom_point(aes('pop_5', 'log_all_streams_cumsum'), color = "red") + ggtitle('Streams [0,100] in Red, Streams [6,100] in black vs Pop 5')
Out[13]:
<ggplot: (-9223363300469049284)>
In [15]:
ggplot(sp, aes('log_cum_sum_5', 'log_cum_sum_after_pop_5')) + geom_point() + geom_abline(slope=1) + ggtitle('Streams[6,100] vs Streams[0,5]')
Out[15]:
<ggplot: (8736385778159)>
In [19]:
ggplot(sp, aes('pop_5','fraction_of_stream_pre_5')) + geom_point()
Out[19]:
<ggplot: (-9223363300469107505)>
In [35]:
model = sm.OLS(sp['log_cum_sum_after_pop_5'], sm.add_constant(sp['pop_5']))
results = model.fit()
results.summary()
Out[35]:
OLS Regression Results
Dep. Variable: log_cum_sum_after_pop_5 R-squared: 0.470
Model: OLS Adj. R-squared: 0.459
Method: Least Squares F-statistic: 45.20
Date: Wed, 05 Sep 2018 Prob (F-statistic): 1.49e-08
Time: 17:01:39 Log-Likelihood: -70.518
No. Observations: 53 AIC: 145.0
Df Residuals: 51 BIC: 149.0
Df Model: 1
Covariance Type: nonrobust
coef std err t P>|t| [0.025 0.975]
const 9.8972 0.501 19.761 0.000 8.892 10.903
pop_5 0.0745 0.011 6.723 0.000 0.052 0.097
Omnibus: 0.888 Durbin-Watson: 2.334
Prob(Omnibus): 0.641 Jarque-Bera (JB): 0.972
Skew: -0.241 Prob(JB): 0.615
Kurtosis: 2.544 Cond. No. 177.


Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.
In [21]:
streams_1_100 = pd.read_feather('../data/archive_as_of_friday_20_july/streams_1_100.feather')
streams_1_100.head()
Out[21]:
track_id cumsum_day_1 cumsum_day_100 cumsum_2to100
0 00kzys67XYXiB31cSP5jfo 26282 473673 447391
1 00mc2RHScEYMEFlc7FRGaK 2141 93919 91778
2 04HzRAn3BJaIvmhpvc1GVT 95386 1426517 1331131
3 04qrVtScdD4IBGSL5q6yEv 54710 2637515 2582805
4 09aaq7feVx9Jykdw0f00QU 58813 1371458 1312645
In [26]:
(ggplot(streams_1_100, aes('cumsum_day_1', 'cumsum_2to100')) + 
geom_point() + 
scale_x_log10() + 
scale_y_log10())
Out[26]:
<ggplot: (-9223363300469275893)>
In [34]:
(ggplot(streams_1_100, aes('cumsum_day_1', 'cumsum_2to100')) + 
geom_point() + 
scale_x_log10() + 
scale_y_log10() + coord_cartesian(xlim=[1, 1e7], ylim=[1,1e7]))
Out[34]:
<ggplot: (8736385176945)>
In [39]:
model = sm.OLS(np.log(streams_1_100['cumsum_2to100']), sm.add_constant(np.log(streams_1_100['cumsum_day_1'])))
results = model.fit()
results.summary()
Out[39]:
OLS Regression Results
Dep. Variable: cumsum_2to100 R-squared: 0.537
Model: OLS Adj. R-squared: 0.528
Method: Least Squares F-statistic: 59.06
Date: Wed, 05 Sep 2018 Prob (F-statistic): 4.53e-10
Time: 17:04:41 Log-Likelihood: -65.431
No. Observations: 53 AIC: 134.9
Df Residuals: 51 BIC: 138.8
Df Model: 1
Covariance Type: nonrobust
coef std err t P>|t| [0.025 0.975]
const 5.8636 0.967 6.063 0.000 3.922 7.805
cumsum_day_1 0.6982 0.091 7.685 0.000 0.516 0.881
Omnibus: 3.598 Durbin-Watson: 2.162
Prob(Omnibus): 0.165 Jarque-Bera (JB): 2.138
Skew: -0.257 Prob(JB): 0.343
Kurtosis: 2.161 Cond. No. 89.2


Warnings:
[1] Standard Errors assume that the covariance matrix of the errors is correctly specified.