import numpy as np import pandas as pd import seaborn as sns import statsmodels.formula.api as sm import statsmodels.imputation.mice as mice from statsmodels.regression.linear_model import OLS from collections import defaultdict import matplotlib.pyplot as plt from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import PolynomialFeatures from sklearn.feature_selection import SelectKBest from sklearn.feature_selection import chi2 #myds = pd.read_csv('catalog_result.csv') #myds = pd.read_csv('physical_only_result.csv') myds = pd.read_csv('latin_cat_result.csv') genres = pd.get_dummies(myds['RELEASE_GENRE']) LABEL_COUNTRY = pd.get_dummies(myds['LABEL_COUNTRY']) df = pd.DataFrame({ "total_rev": myds['TOTAL_GROSS_REVENUE_USD_LAST_YEAR'], "rev_recip": np.reciprocal(myds['TOTAL_GROSS_REVENUE_USD_LAST_YEAR']), "rev_log": np.log(myds['TOTAL_GROSS_REVENUE_USD_LAST_YEAR']), "total_physical_rev": myds['PHYSICAL_GROSS_REVENUE_LAST_YEAR'], #"num_cds": myds['NUM_OF_CDS'], #NOT LY DATA! "num_active_releases": myds['NUM_ACTIVE_RELEASES'], "cd_units": myds['CD_REVENUE_UNITS_SOLD_LAST_YEAR'], "spotify_passive_streams": myds['SPOTIFY_STREAMS_PASSIVE_DISC_LAST_YEAR'], "spotify_playlist_streams": myds['SPOTIFY_PLAYLISTS_STREAMS_LAST_YEAR'], "spotify_active_streams": myds['SPOTIFY_STREAMS_ACTIVE_DISC_LAST_YEAR'], "spotify_collection_streams": myds['SPOTIFY_STREAMS_COLLECTION_LAST_YEAR'], "spotify_total_streams": myds['SPOTIFY_OVERALL_STREAMS_LAST_YEAR'], "spotify_passive_percent": myds['SPOTIFY_STREAMS_PASSIVE_DISC_LAST_YEAR']/myds['SPOTIFY_OVERALL_STREAMS_LAST_YEAR'], "spotify_playlist_percent": myds['SPOTIFY_PLAYLISTS_STREAMS_LAST_YEAR']/myds['SPOTIFY_OVERALL_STREAMS_LAST_YEAR'], "spotify_active_percent": myds['SPOTIFY_STREAMS_ACTIVE_DISC_LAST_YEAR']/myds['SPOTIFY_OVERALL_STREAMS_LAST_YEAR'], "spotify_collection_percent": myds['SPOTIFY_STREAMS_COLLECTION_LAST_YEAR']/myds['SPOTIFY_OVERALL_STREAMS_LAST_YEAR'], "apple_playlist_streams": myds['APPLE_PLAYLISTS_STREAMS_LAST_YEAR'], "apple_passive_streams": myds['APPLE_STREAMS_PASSIVE_DISC_LAST_YEAR'], "apple_active_streams": myds['APPLE_STREAMS_ACTIVE_DISC_LAST_YEAR'], "apple_collection_streams": myds['APPLE_STREAMS_COLLECTION_LAST_YEAR'], "apple_total_streams": myds['APPLE_OVERALL_STREAMS_LAST_YEAR'], "apple_passive_percent": myds['APPLE_STREAMS_PASSIVE_DISC_LAST_YEAR']/myds['APPLE_OVERALL_STREAMS_LAST_YEAR'], "apple_playlist_percent": myds['APPLE_PLAYLISTS_STREAMS_LAST_YEAR']/myds['APPLE_OVERALL_STREAMS_LAST_YEAR'], "apple_active_percent": myds['APPLE_STREAMS_ACTIVE_DISC_LAST_YEAR']/myds['APPLE_OVERALL_STREAMS_LAST_YEAR'], "apple_collection_percent": myds['APPLE_STREAMS_COLLECTION_LAST_YEAR']/myds['APPLE_OVERALL_STREAMS_LAST_YEAR'], "amazon_total_streams": myds['AMAZON_OVERALL_STREAMS_LAST_YEAR'], "pandora_total_streams": myds['PANDORA_OVERALL_STREAMS_LAST_YEAR'], "youtube_audio_streams": myds['YOUTUBE_AUDIO_OVERALL_STREAMS_LAST_YEAR'], "shazam_total_streams": myds['SHAZAM_OVERALL_STREAMS_LAST_YEAR'], "alexa_total_streams": myds['ALEXA_OVERALL_STREAMS_LAST_YEAR'], "itunes_total_downloads": myds['ITUNES_TOTAL_DOWNLOADS_LAST_YEAR'], "amazon_total_downloads": myds['AMAZON_TOTAL_DOWNLOADS_LAST_YEAR'], "google_total_downloads": myds['GOOGLE_PLAY_TOTAL_DOWNLOADS_LAST_YEAR'], }) # df_soc = pd.DataFrame({ # "fb_likes": myds['FB_LIKES_LTD'], # "twitter_followers": myds['TWITTER_FOLLOWERS_LTD'], # "youtube_subs": myds['YOUTUBE_SUBS_LTD'], # "wiki_pageviews": myds['WIKI_PAGEVIEWS_LTD'], # "insta_followers": myds['INSTA_FOLLOWERS_LTD'], # "spotify_followers": myds['SPOTIFY_FOLLOWERS_TD'], # "fb_talking_about": myds['FB_TALKING_ABOUT_TD'], # "vevo_views": myds['VEVO_VIDEO_VIEWS_TD'], # "twitter_mentions": myds['TWITTER_MENTIONS_TD'], # "twitter_retweets": myds['TWITTER_RETWEETS_TD'], # "youtube_likes": myds['YOUTUBE_LIKES_TD'], # "insta_comments": myds['INSTAGRAM_COMMENTS_TD'], # "songkick_followers": myds['SONGKICK_FOLLOWERS_LTD'], # }) #impute NaN with 0s df = df.fillna(0) #all_vars = pd.concat([df.fillna(0), genres, LABEL_COUNTRY, df_soc["songkick_followers"],df_soc["fb_likes"],df_soc["twitter_followers"]] , axis=1) #genre_streams = pd.concat([df["apple_collection_percent"], genres] , axis=1) #gs_corr = genre_streams.corr() #print(gs_corr) av = pd.concat([df, genres, LABEL_COUNTRY] , axis=1) av1 = pd.concat([av, LABEL_COUNTRY] , axis=1) # corr2 = df.corr() # print(corr2) # sns.heatmap(corr2) # plots # # plot1 = av1.plot(y="total_rev",x="apple_total_streams",kind='scatter') # plot1.set_ylim(0,200000) # plot1.set_xlim(0,500000) # # plot2 = av1.plot(y="total_rev",x="alexa_total_streams",kind='scatter') # plot2.set_ylim(0,300000) # plot2.set_xlim(0,100000) # #plot3 = av1.hist("total_rev", bins=100) # plot4 = av1.hist("rev_log", bins=100) # plot5 = av1.hist("rev_recip", bins=100) # # plot6 = av1.plot(y="total_rev",x="spotify_playlist_streams",kind='scatter') # # # print("Plot 1:") #print(plot3) #plot2, plot3, plot4, plot5, plot6) #print plt.hist(np.reciprocal(myds['TOTAL_GROSS_REVENUE']), bins=100) #print plt.hist(myds['TOTAL_GROSS_REVENUE'], bins=100,range=(10000,1000000)) #fml = "total_rev ~ total_physical_rev + cd_units + num_active_releases + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" fml = "total_rev ~ spotify_total_streams + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + apple_total_streams + apple_playlist_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "total_rev ~ spotify_total_streams + spotify_playlist_percent + spotify_active_percent + spotify_passive_percent + spotify_collection_percent + apple_total_streams + apple_playlist_percent + apple_active_percent + apple_passive_percent + apple_collection_percent + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "total_rev ~ spotify_total_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "rev_recip ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "rev_log ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + spotify_total_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #formula for streaming: #fml = "apple_total_streams ~ apple_passive_percent + apple_playlist_percent + apple_active_percent + apple_collection_percent" #fml = "spotify_total_streams ~ spotify_passive_percent + spotify_playlist_percent + spotify_active_percent + spotify_collection_percent" result = sm.ols(fml, data=av).fit() print(result.summary()) print(df["spotify_playlist_percent"].mean()) #plt.show()