import numpy as np
import pandas as pd
import seaborn as sns
import statsmodels.formula.api as sm
import statsmodels.imputation.mice as mice
from statsmodels.regression.linear_model import OLS
from collections import defaultdict
import matplotlib.pyplot as plt
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import PolynomialFeatures
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2
from math import log


ly_avg = pd.read_csv('tt_daily_avg_ly_april16.csv', encoding = "ISO-8859-1")
today = pd.read_csv('tt_today_april16.csv', encoding = "ISO-8859-1")
rstory = pd.read_csv('tt_reddit_story_ranking_april16.csv', encoding = "ISO-8859-1")

all_tracks = pd.merge(ly_avg,today, on=['TRACKID','TRACKNAME','ARTISTNAME'])
all_tracks = pd.merge(all_tracks,rstory, on=['TRACKNAME','ARTISTNAME'])


#print(all_tracks[:10])

all_tracks['zscore'] = (all_tracks['TODAY']-all_tracks['AVG(SUMS)'])/all_tracks['STDDEV(SUMS)']


conditions = [
    (all_tracks['zscore'] >0),
    (all_tracks['zscore'] <0),
    (all_tracks['zscore'] ==0)]
choices = [1, -1, 0]
all_tracks['y1'] = np.select(conditions, choices)

conditions1 = [
    (abs(all_tracks['zscore']) >=1),
    (abs(all_tracks['zscore']) <1)]
    
choices1 = [abs(all_tracks['zscore']), 1]
all_tracks['z1'] = np.select(conditions1, choices1)

all_tracks['logz1'] = all_tracks['z1'].apply(np.log10) 

all_tracks['sumscore'] = all_tracks['SCORE'] + all_tracks['y1']*all_tracks['logz1']

all_tracks = all_tracks.sort_values('sumscore',ascending=False)
# 
print(all_tracks[:50])
# 
all_tracks.to_csv('tt_sumscore_april16.csv')