import numpy as np import pandas as pd import seaborn as sns import statsmodels.formula.api as sm import statsmodels.imputation.mice as mice from statsmodels.regression.linear_model import OLS from collections import defaultdict import matplotlib.pyplot as plt from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import PolynomialFeatures from sklearn.feature_selection import SelectKBest from sklearn.feature_selection import chi2 myds = pd.read_csv('catalog_result.csv') #myds = pd.read_csv('catalog_result_socials.csv', encoding = "ISO-8859-1") #myds = pd.read_csv('latin_cat_result.csv') genres = pd.get_dummies(myds['RELEASE_GENRE']) LABEL_COUNTRY = pd.get_dummies(myds['LABEL_COUNTRY']) df = pd.DataFrame({ "total_rev": myds['TOTAL_GROSS_REVENUE'], "rev_recip": np.reciprocal(myds['TOTAL_GROSS_REVENUE']), "rev_log": np.log(myds['TOTAL_GROSS_REVENUE']), "total_physical_rev": myds['PHYSICAL_GROSS_REVENUE'], "num_cds": myds['NUM_OF_CDS'], "num_active_releases": myds['NUM_ACTIVE_RELEASES'], "cd_units": myds['CD_REVENUE_UNITS_SOLD'], "spotify_passive_streams": myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL'], "spotify_playlist_streams": myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL'], "spotify_active_streams": myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL'], "spotify_collection_streams": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL'], "spotify_total_streams": myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_passive_percent": myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_playlist_percent": myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_active_percent": myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_collection_percent": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "apple_passive_streams": myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL'], "apple_playlist_streams": myds['APPLE_PLAYLISTS_STREAMS_TOTAL'], "apple_active_streams": myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL'], "apple_collection_streams": myds['APPLE_STREAMS_COLLECTION_TOTAL'], "apple_total_streams": myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_passive_percent": myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_playlist_percent": myds['APPLE_PLAYLISTS_STREAMS_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_active_percent": myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_collection_percent": myds['APPLE_STREAMS_COLLECTION_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "amazon_total_streams": myds['AMAZON_OVERALL_STREAMS_TOTAL'], "pandora_total_streams": myds['PANDORA_OVERALL_STREAMS_TOTAL'], "youtube_audio_streams": myds['YOUTUBE_AUDIO_OVERALL_STREAMS_TOTAL'], "shazam_total_streams": myds['SHAZAM_OVERALL_STREAMS_TOTAL'], "alexa_total_streams": myds['ALEXA_OVERALL_STREAMS_TOTAL'], "itunes_total_downloads": myds['ITUNES_TOTAL_DOWNLOADS'], "amazon_total_downloads": myds['AMAZON_TOTAL_DOWNLOADS'], "google_total_downloads": myds['GOOGLE_PLAY_DOWNLOADS_TOTAL'], }) df_soc = pd.DataFrame({ "fb_likes": myds['FB_LIKES_LTD'], "twitter_followers": myds['TWITTER_FOLLOWERS_LTD'], "youtube_subs": myds['YOUTUBE_SUBS_LTD'], #"wiki_pageviews": myds['WIKI_PAGEVIEWS_LTD'], "insta_followers": myds['INSTA_FOLLOWERS_LTD'], "spotify_followers": myds['SPOTIFY_FOLLOWERS_LTD'], "fb_talking_about": myds['FB_TALKING_ABOUT_LY'], #"vevo_views": myds['VEVO_VIDEO_VIEWS_LTD'], "twitter_mentions": myds['TWITTER_MENTIONS_LY'], "twitter_retweets": myds['TWITTER_RETWEETS_LY'], "youtube_likes": myds['YOUTUBE_LIKES_LTD'], "insta_comments": myds['INSTAGRAM_COMMENTS_LTD'], "songkick_followers": myds['SONGKICK_FOLLOWERS_LTD'], }) #print(df_soc.isnull().sum()) print(df_soc.count()) # df_soc_cut = df_soc.dropna() # plot1 = df_soc_cut.hist("insta_followers", bins=100) #impute NaN with 0s df = df.fillna(0) genres = genres.fillna(0) LABEL_COUNTRY = LABEL_COUNTRY.fillna(0) #set negative values to 0 df[df < 0] = 0 print("df size:") print(df.shape) df_soc[df_soc < 0] = 0 #create full all_vars table all_vars = pd.concat([df.fillna(0), genres, LABEL_COUNTRY, df_soc] , axis=1) all_vars1 = pd.concat([df.fillna(0), df_soc] , axis=1) #create partial socials table #all_vars = pd.concat([df.fillna(0), genres, LABEL_COUNTRY, df_soc["songkick_followers"],df_soc["fb_likes"],df_soc["twitter_followers"]] , axis=1) all_soc_vars = pd.concat([df_soc, df["total_rev"]] , axis=1) #corr1 = all_vars1.corr() #print(corr1) #print(sns.heatmap(corr1)) print("shape with social NaNs:") print(all_vars.shape) #drop social NaNs #all_vars = all_vars.dropna() # print("shape with dropped NaNs:") # print(all_vars.shape) # corr2 = all_vars.corr() # print(corr2) # print(sns.heatmap(corr2)) #formula without socials # social formula fml = "total_rev ~ num_active_releases + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + spotify_total_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads + fb_likes + insta_followers + twitter_followers + youtube_subs + songkick_followers + twitter_retweets" fml_playlist = "spotify_collection_streams ~ fb_likes + fb_talking_about + insta_followers + insta_comments + twitter_followers + twitter_retweets + youtube_subs" #fml_soc = "total_rev ~ fb_likes + fb_talking_about + insta_followers + insta_comments + spotify_followers + twitter_followers + twitter_mentions + twitter_retweets + youtube_likes + youtube_subs + songkick_followers" fml_blocparty = "total_rev ~ num_active_releases + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + spotify_total_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads + fb_likes + fb_talking_about + insta_followers + insta_comments + twitter_followers + twitter_retweets + youtube_subs + songkick_followers" # result = sm.ols(fml_playlist, data=all_vars).fit() # print(result.summary()) result = sm.ols(fml, data=all_vars).fit() print(result.summary()) plt.show()