import numpy as np
import pandas as pd
import seaborn as sns
import statsmodels.formula.api as sm
import statsmodels.imputation.mice as mice
from statsmodels.regression.linear_model import OLS
from collections import defaultdict
import matplotlib.pyplot as plt
from sklearn.feature_selection import VarianceThreshold
from sklearn.preprocessing import PolynomialFeatures
from sklearn.feature_selection import SelectKBest
from sklearn.feature_selection import chi2


myds = pd.read_csv('catalog_result.csv')


genres = pd.get_dummies(myds['RELEASE_GENRE'])
LABEL_COUNTRY = pd.get_dummies(myds['LABEL_COUNTRY'])


#LOGS!!!!!!!!!!!!!!!!
df = pd.DataFrame({
"total_rev": myds['TOTAL_GROSS_REVENUE'],
  "rev_recip": np.reciprocal(myds['TOTAL_GROSS_REVENUE']),
  "rev_log": np.log(myds['TOTAL_GROSS_REVENUE']),
  
"log_total_physical_rev": np.log(myds['PHYSICAL_GROSS_REVENUE']),
"log_num_cds": np.log(myds['NUM_OF_CDS']),
"log_num_active_releases": np.log(myds['NUM_ACTIVE_RELEASES']),
"log_cd_revenue": np.log(myds['CD_REVENUE_UNITS_SOLD']),
"log_spotify_playlist_streams": np.log(myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL']),
"log_spotify_active_streams": np.log(myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL']),
"log_spotify_collection_streams": np.log(myds['SPOTIFY_STREAMS_COLLECTION_TOTAL']),
"log_spotify_passive_streams": np.log(myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL']),
"log_spotify_total_streams": np.log(myds['SPOTIFY_OVERALL_STREAMS_TOTAL']),
"log_apple_active_streams": np.log(myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL']),
"log_apple_collection_streams": np.log(myds['APPLE_STREAMS_COLLECTION_TOTAL']),
"log_apple_passive_streams": np.log(myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL']),
"log_apple_total_streams": np.log(myds['APPLE_OVERALL_STREAMS_TOTAL']),
"log_amazon_total_streams": np.log(myds['AMAZON_OVERALL_STREAMS_TOTAL']),
"log_pandora_total_streams": np.log(myds['PANDORA_OVERALL_STREAMS_TOTAL']),
"log_youtube_audio_streams": np.log(myds['YOUTUBE_AUDIO_OVERALL_STREAMS_TOTAL']),
"log_shazam_total_streams": np.log(myds['SHAZAM_OVERALL_STREAMS_TOTAL']),
"log_alexa_total_streams": np.log(myds['ALEXA_OVERALL_STREAMS_TOTAL']),
"log_itunes_total_downloads": np.log(myds['ITUNES_TOTAL_DOWNLOADS']),
"log_amazon_total_downloads": np.log(myds['AMAZON_TOTAL_DOWNLOADS']),
"log_google_total_downloads": np.log(myds['GOOGLE_PLAY_DOWNLOADS_TOTAL']),
})

#impute NaN with 0s
df = df.fillna(0)
#set negative values to 0
df[df < 0] = 0

#DROP ALL ZEROS (for plotting)
df = df.loc[(df!=0).any(axis=1)]


av = pd.concat([df, genres, LABEL_COUNTRY] , axis=1)
av1 = pd.concat([av, LABEL_COUNTRY] , axis=1)

corr2 = df.corr()
#print corr2
print(sns.heatmap(corr2))

#plots

plot1 = df.plot(y="total_rev",x="log_apple_total_streams",kind='scatter')
#plot1.set_ylim(0,200000)
#plot1.set_xlim(0,500000)

plot2 = df.plot(y="total_rev",x="log_alexa_total_streams",kind='scatter')
plot2.set_ylim(0,300000)
plot2.set_xlim(0,100000)

plot3 = df.hist("total_rev", bins=100)
plot4 = df.hist("rev_log", bins=100)
plot5 = df.hist("rev_recip", bins=100)

plot6 = df.plot(y="rev_log",x="log_spotify_playlist_streams",kind='scatter')
plot7 = df.plot(y="rev_log",x="log_spotify_active_streams",kind='scatter')
plot8 = df.plot(y="rev_log",x="log_spotify_collection_streams",kind='scatter')
plot9 = df.plot(y="rev_log",x="log_spotify_total_streams",kind='scatter')



print(plot1, plot2, plot3, plot4, plot5, plot6, plot7, plot8, plot9)

#print plt.hist(np.reciprocal(np.log(myds['TOTAL_GROSS_REVENUE']), bins=100)
#print plt.hist(np.log(myds['TOTAL_GROSS_REVENUE'], bins=100,range=(10000,1000000))



#fml = "total_rev ~ total_physical_rev   + num_cds + num_active_releases + cd_revenue  + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads"
#fml = "rev_recip ~ total_physical_rev   + num_cds + num_active_releases + cd_revenue  + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads"
fml = "rev_log ~ log_total_physical_rev   + log_num_cds + log_num_active_releases + log_cd_revenue  + log_spotify_playlist_streams + log_spotify_active_streams + log_spotify_collection_streams + log_spotify_passive_streams + log_spotify_total_streams  + log_apple_active_streams + log_apple_collection_streams + log_apple_passive_streams + log_apple_total_streams + log_amazon_total_streams + log_pandora_total_streams + log_shazam_total_streams + log_alexa_total_streams + log_youtube_audio_streams + log_itunes_total_downloads + log_amazon_total_downloads + log_google_total_downloads"




result = sm.ols(fml, data=av1).fit()
print(result.summary())


plt.show()