import numpy as np import pandas as pd import seaborn as sns import statsmodels.formula.api as sm import statsmodels.imputation.mice as mice from statsmodels.regression.linear_model import OLS from collections import defaultdict import matplotlib.pyplot as plt from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import PolynomialFeatures from sklearn.feature_selection import SelectKBest from sklearn.feature_selection import chi2 myds = pd.read_csv('catalog_result.csv') genres = pd.get_dummies(myds['RELEASE_GENRE']) LABEL_COUNTRY = pd.get_dummies(myds['LABEL_COUNTRY']) #LOGS!!!!!!!!!!!!!!!! df = pd.DataFrame({ "total_rev": myds['TOTAL_GROSS_REVENUE'], "rev_recip": np.reciprocal(myds['TOTAL_GROSS_REVENUE']), "rev_log": np.log(myds['TOTAL_GROSS_REVENUE']), "log_total_physical_rev": np.log(myds['PHYSICAL_GROSS_REVENUE']), "log_num_cds": np.log(myds['NUM_OF_CDS']), "log_num_active_releases": np.log(myds['NUM_ACTIVE_RELEASES']), "log_cd_revenue": np.log(myds['CD_REVENUE_UNITS_SOLD']), "log_spotify_playlist_streams": np.log(myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL']), "log_spotify_active_streams": np.log(myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL']), "log_spotify_collection_streams": np.log(myds['SPOTIFY_STREAMS_COLLECTION_TOTAL']), "spotify_collection_streams": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL'], "log_spotify_passive_streams": np.log(myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL']), "log_spotify_total_streams": np.log(myds['SPOTIFY_OVERALL_STREAMS_TOTAL']), "spotify_total_streams": myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "log_apple_playlist_streams": np.log(myds['APPLE_PLAYLISTS_STREAMS_TOTAL']), "log_apple_active_streams": np.log(myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL']), "log_apple_collection_streams": np.log(myds['APPLE_STREAMS_COLLECTION_TOTAL']), "apple_collection_streams": myds['APPLE_STREAMS_COLLECTION_TOTAL'], "log_apple_passive_streams": np.log(myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL']), "log_apple_total_streams": np.log(myds['APPLE_OVERALL_STREAMS_TOTAL']), "log_amazon_total_streams": np.log(myds['AMAZON_OVERALL_STREAMS_TOTAL']), "log_pandora_total_streams": np.log(myds['PANDORA_OVERALL_STREAMS_TOTAL']), "log_youtube_audio_streams": np.log(myds['YOUTUBE_AUDIO_OVERALL_STREAMS_TOTAL']), "log_shazam_total_streams": np.log(myds['SHAZAM_OVERALL_STREAMS_TOTAL']), "log_alexa_total_streams": np.log(myds['ALEXA_OVERALL_STREAMS_TOTAL']), "log_itunes_total_downloads": np.log(myds['ITUNES_TOTAL_DOWNLOADS']), "log_amazon_total_downloads": np.log(myds['AMAZON_TOTAL_DOWNLOADS']), "log_google_total_downloads": np.log(myds['GOOGLE_PLAY_DOWNLOADS_TOTAL']), }) df_soc = pd.DataFrame({ "log_fb_likes": np.log(myds['FB_LIKES_LTD']), "log_twitter_followers": np.log(myds['TWITTER_FOLLOWERS_LTD']), "log_youtube_subs": np.log(myds['YOUTUBE_SUBS_LTD']), #"wiki_pageviews": myds['WIKI_PAGEVIEWS_LTD'], "log_insta_followers": np.log(myds['INSTA_FOLLOWERS_LTD']), "log_spotify_followers": np.log(myds['SPOTIFY_FOLLOWERS_LTD']), "log_fb_talking_about": np.log(myds['FB_TALKING_ABOUT_TD']), #"vevo_views": myds['VEVO_VIDEO_VIEWS_LTD'], "log_twitter_mentions": np.log(myds['TWITTER_MENTIONS_TD']), "log_twitter_retweets": np.log(myds['TWITTER_RETWEETS_TD']), "log_youtube_likes": np.log(myds['YOUTUBE_LIKES_LTD']), "log_insta_comments": np.log(myds['INSTAGRAM_COMMENTS_LTD']), "log_songkick_followers": np.log(myds['SONGKICK_FOLLOWERS_LTD']), }) #impute NaN with 0s df = df.fillna(0) #set negative values to 0 df[df < 0] = 0 df_soc[df_soc < 0] = 0 #DROP ALL ZEROS (for plotting) df = df.loc[(df!=0).any(axis=1)] df_soc = df_soc.loc[(df!=0).any(axis=1)] av = pd.concat([df, genres, LABEL_COUNTRY, df_soc] , axis=1) av1 = pd.concat([av, LABEL_COUNTRY] , axis=1) #corr2 = df.corr() #print corr2 #print(sns.heatmap(corr2)) #plots # plot1 = df.plot(y="total_rev",x="log_apple_total_streams",kind='scatter') # #plot1.set_ylim(0,200000) # #plot1.set_xlim(0,500000) # # plot2 = df.plot(y="total_rev",x="log_alexa_total_streams",kind='scatter') # plot2.set_ylim(0,300000) # plot2.set_xlim(0,100000) # # plot3 = df.hist("total_rev", bins=100) # plot4 = df.hist("rev_log", bins=100) # plot5 = df.hist("rev_recip", bins=100) # # plot6 = df.plot(y="rev_log",x="log_spotify_playlist_streams",kind='scatter') # plot7 = df.plot(y="rev_log",x="log_spotify_active_streams",kind='scatter') # plot8 = df.plot(y="rev_log",x="log_spotify_collection_streams",kind='scatter') # plot9 = df.plot(y="rev_log",x="log_spotify_total_streams",kind='scatter') # # # # print(plot1, plot2, plot3, plot4, plot5, plot6, plot7, plot8, plot9) #print plt.hist(np.reciprocal(np.log(myds['TOTAL_GROSS_REVENUE']), bins=100) #print plt.hist(np.log(myds['TOTAL_GROSS_REVENUE'], bins=100,range=(10000,1000000)) #fml = "total_rev ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "rev_recip ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "rev_log ~ log_spotify_playlist_streams + log_spotify_active_streams + log_spotify_collection_streams + log_spotify_passive_streams + log_spotify_total_streams + log_apple_playlist_streams + log_apple_active_streams + log_apple_collection_streams + log_apple_passive_streams + log_apple_total_streams + log_amazon_total_streams + log_pandora_total_streams + log_shazam_total_streams + log_alexa_total_streams + log_youtube_audio_streams + log_itunes_total_downloads + log_amazon_total_downloads + log_google_total_downloads + Rock + USA" #fml = "rev_log ~ log_spotify_total_streams + log_apple_total_streams + log_amazon_total_streams + log_pandora_total_streams + log_shazam_total_streams + log_alexa_total_streams + log_youtube_audio_streams + log_itunes_total_downloads + log_amazon_total_downloads + log_google_total_downloads" #bloc_party_fml = "total_rev ~ log_spotify_total_streams + log_apple_total_streams + log_amazon_total_streams + log_pandora_total_streams + log_shazam_total_streams + log_alexa_total_streams + log_youtube_audio_streams + log_itunes_total_downloads + log_amazon_total_downloads + log_google_total_downloads + log_fb_talking_about + log_twitter_retweets + log_twitter_mentions + log_insta_comments + log_twitter_followers + Rock + USA" #bloc_party_fml = "spotify_total_streams ~ log_fb_likes + log_fb_talking_about + log_twitter_retweets + log_twitter_mentions + log_insta_followers + log_insta_comments + log_twitter_followers + Rock + USA" huskerdu_fml = "apple_collection_streams ~ log_fb_likes + log_fb_talking_about + log_twitter_retweets + log_twitter_mentions + log_insta_followers + log_insta_comments + log_twitter_followers + Rock + USA" #log_spotify_playlist_streams + log_spotify_active_streams + log_spotify_collection_streams + log_spotify_passive_streams + log_apple_playlist_streams + log_apple_active_streams + log_apple_collection_streams + log_apple_passive_streams + result = sm.ols(huskerdu_fml, data=av).fit() print(result.summary()) plt.show()