Exploratory analysis of streams and popularity

Specifically trying to understand why the total streams is pracitcally limited downwards for the same pop

In [3]:
import pandas as pd
import numpy as np
%matplotlib inline
%pylab inline
pylab.rcParams['figure.figsize'] = (12, 7)

from plotnine import *

Populating the interactive namespace from numpy and matplotlib
In [4]:
    pop = pd.read_feather('../data/pop_with_streams.feather').drop('index', axis = 1)
    release_dates = pd.read_feather('../data/release_dates.feather')

    pop_rel = pd.merge(pop, release_dates, left_on='track_spyid', right_on='spyid')

    pop_rel['days_since_release']=((pop_rel['as_of']-pop_rel['release_date'])/pd.Timedelta('1 d'))

    #TODO: Why are these 
    pop_rel = pop_rel[~pop_rel['days_since_release'].isnull()]
    pop_rel['days_since_release'] = pop_rel['days_since_release'].astype(int)


    pop_rel['diff_release_first_seen']=((pop_rel['first_seen']-pop_rel['release_date'])/pd.Timedelta('1 d'))
In [5]:
(ggplot(pop_rel[pop_rel['days_since_release'] < 10], aes('days_since_release', 'value', group = 'spyid')) + geom_line(alpha=0.3))

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/utils.py:281: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  ndistinct = ids.apply(len_unique, axis=0).as_matrix()
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4388: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  object.__getattribute__(self, name)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4389: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  return object.__setattr__(self, name, value)
Out[5]:
<ggplot: (8746420756337)>
In [6]:
ggplot(pop, aes('as_of', 'value')) + geom_point(alpha = 0.1)

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4388: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  object.__getattribute__(self, name)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4389: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  return object.__setattr__(self, name, value)
Out[6]:
<ggplot: (8746421208552)>
In [7]:
ggplot(pop, aes('as_of')) + geom_histogram()

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 142'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4388: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  object.__getattribute__(self, name)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4389: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  return object.__setattr__(self, name, value)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/positions/position.py:188: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  intervals = data[xminmax].drop_duplicates().as_matrix().flatten()
Out[7]:
<ggplot: (-9223363290487498822)>
In [8]:
 streams = pd.read_feather('../data/raw_stream.feather')
In [9]:
streams.dtypes
Out[9]:
spyid              object
as_of      datetime64[ns]
streams             int64
dtype: object
In [10]:
len(streams)
Out[10]:
2766
In [11]:
ggplot(streams, aes('as_of', 'streams')) + geom_point(alpha = 0.1)

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4388: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  object.__getattribute__(self, name)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4389: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  return object.__setattr__(self, name, value)
Out[11]:
<ggplot: (8746401329486)>
In [12]:
ggplot(streams, aes('as_of')) + geom_histogram()

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 27'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4388: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  object.__getattribute__(self, name)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4389: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  return object.__setattr__(self, name, value)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/positions/position.py:188: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  intervals = data[xminmax].drop_duplicates().as_matrix().flatten()
Out[12]:
<ggplot: (8746369079749)>

Explore pop in the beginning of the series

In [21]:
pop = pd.read_feather('../data/pop_with_streams.feather').drop('index', axis = 1)
pop.set_index('as_of')
for start in np.arange(5):
    pp = pop.groupby('track_spyid').apply(lambda x: x[start:(start + 1)])
    zero_pp = len(pp[pp['value'] == 0])/len(pp)
    print(ggplot(pp, aes('value')) + geom_histogram() + ggtitle(f'Start: {start}. Percent 0: {zero_pp:0.2f}'))

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 17'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4388: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  object.__getattribute__(self, name)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/pandas/core/generic.py:4389: FutureWarning: Attribute 'is_copy' is deprecated and will be removed in a future version.
  return object.__setattr__(self, name, value)
/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/positions/position.py:188: FutureWarning: Method .as_matrix will be removed in a future version. Use .values instead.
  intervals = data[xminmax].drop_duplicates().as_matrix().flatten()

<ggplot: (-9223363290453441960)>

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 21'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))

<ggplot: (8746404490237)>

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 21'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))

<ggplot: (-9223363290485923780)>

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 30'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))

<ggplot: (-9223363290485923731)>

/home/paperspace/anaconda3/envs/whitelist/lib/python3.6/site-packages/plotnine/stats/stat_bin.py:90: UserWarning: 'stat_bin()' using 'bins = 30'. Pick better value with 'binwidth'.
  warn(msg.format(params['bins']))

<ggplot: (8746401351211)>
In [18]:
start = 0
pop_0 = pop.groupby('track_spyid').apply(lambda x: x[start:(start + 1)])
len(pop_0[pop_0['value']==0])/len(pop_0)
Out[18]:
0.22846580406654343