import numpy as np
import pandas as pd
import seaborn as sns
import statsmodels.formula.api as sm
import statsmodels.imputation.mice as mice
from statsmodels.regression.linear_model import OLS
from collections import defaultdict
import matplotlib.pyplot as plt
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import PolynomialFeatures
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2


diff = pd.read_csv('fp_today_minus_yesterday_March21.csv', encoding = "ISO-8859-1")
avg = pd.read_csv('fp_avg_diff_March21.csv', encoding = "ISO-8859-1")

diff = diff.dropna()

all_tracks = pd.merge(diff,avg, on=['TRACKID','TRACKNAME','ARTISTNAME'])

print(all_tracks[:10])

all_tracks['zscore'] = (all_tracks['DIFF']-all_tracks['AVG_DIFF'])/all_tracks['STD_DEV_DIFF']

all_tracks = all_tracks.sort_values('zscore',ascending=False)
# 
print(all_tracks[:50])
# 
all_tracks.to_csv('fp_zscoreDIFF_trending_March21.csv')