import numpy as np import pandas as pd import seaborn as sns import statsmodels.formula.api as sm import statsmodels.imputation.mice as mice from statsmodels.regression.linear_model import OLS from collections import defaultdict import matplotlib.pyplot as plt from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import PolynomialFeatures from sklearn.feature_selection import SelectKBest from sklearn.feature_selection import chi2 myds = pd.read_csv('catalog_result.csv') #PHYSICAL ONLY: #myds = pd.read_csv('physical_only_result.csv') #myds = pd.read_csv('latin_cat_result.csv') genres = pd.get_dummies(myds['RELEASE_GENRE']) LABEL_COUNTRY = pd.get_dummies(myds['LABEL_COUNTRY']) df = pd.DataFrame({ #"aaartist_name": myds['ARTIST_NAME'], "total_rev": myds['TOTAL_GROSS_REVENUE'], "rev_recip": np.reciprocal(myds['TOTAL_GROSS_REVENUE']), "rev_log": np.log(myds['TOTAL_GROSS_REVENUE']), "total_physical_rev": myds['PHYSICAL_GROSS_REVENUE'], "num_cds": myds['NUM_OF_CDS'], "num_active_releases": myds['NUM_ACTIVE_RELEASES'], "cd_units": myds['CD_REVENUE_UNITS_SOLD'], "spotify_passive_streams": myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL'], "spotify_playlist_streams": myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL'], "spotify_active_streams": myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL'], "spotify_collection_streams": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL'], "spotify_total_streams": myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_passive_percent": myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_playlist_percent": myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_active_percent": myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_collection_percent": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "apple_passive_streams": myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL'], "apple_playlist_streams": myds['APPLE_PLAYLISTS_STREAMS_TOTAL'], "apple_active_streams": myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL'], "apple_collection_streams": myds['APPLE_STREAMS_COLLECTION_TOTAL'], "apple_total_streams": myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_passive_percent": myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_playlist_percent": myds['APPLE_PLAYLISTS_STREAMS_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_active_percent": myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_collection_percent": myds['APPLE_STREAMS_COLLECTION_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "amazon_total_streams": myds['AMAZON_OVERALL_STREAMS_TOTAL'], "pandora_total_streams": myds['PANDORA_OVERALL_STREAMS_TOTAL'], "youtube_audio_streams": myds['YOUTUBE_AUDIO_OVERALL_STREAMS_TOTAL'], "shazam_total_streams": myds['SHAZAM_OVERALL_STREAMS_TOTAL'], "alexa_total_streams": myds['ALEXA_OVERALL_STREAMS_TOTAL'], "itunes_total_downloads": myds['ITUNES_TOTAL_DOWNLOADS'], "amazon_total_downloads": myds['AMAZON_TOTAL_DOWNLOADS'], "google_total_downloads": myds['GOOGLE_PLAY_DOWNLOADS_TOTAL'], }) # df_soc = pd.DataFrame({ # "fb_likes": myds['FB_LIKES_LTD'], # "twitter_followers": myds['TWITTER_FOLLOWERS_LTD'], # "youtube_subs": myds['YOUTUBE_SUBS_LTD'], # "wiki_pageviews": myds['WIKI_PAGEVIEWS_LTD'], # "insta_followers": myds['INSTA_FOLLOWERS_LTD'], # "spotify_followers": myds['SPOTIFY_FOLLOWERS_TD'], # }) #impute NaN with 0s df = df.fillna(0) # genre_streams = pd.concat([df["spotify_collection_percent"], genres] , axis=1) # gs_corr = genre_streams.corr() # print(gs_corr) av = pd.concat([df, genres, LABEL_COUNTRY] , axis=1) #av1 = pd.concat([av, LABEL_COUNTRY] , axis=1) # corr2 = df.corr() # print(corr2) # sns.heatmap(corr2) #plots # plot1 = av1.plot(y="total_rev",x="apple_total_streams",kind='scatter') # #plot1.set_ylim(0,200000) # #plot1.set_xlim(0,500000) # # plot2 = av1.plot(y="total_rev",x="alexa_total_streams",kind='scatter') # plot2.set_ylim(0,300000) # plot2.set_xlim(0,100000) # # plot3 = av.hist("total_rev", bins=1000) # # plot4 = av.hist("rev_log", bins=1000) # plot5 = av1.hist("rev_recip", bins=100) # #Spotify streams plot # ax = av1.plot(y="total_rev",x="apple_playlist_streams",kind='scatter', color='DarkBlue',label='Playlist') # av1.plot(y="total_rev",x="apple_active_streams",kind='scatter', color='DarkGreen', label='Active', ax=ax) # av1.plot(y="total_rev",x="apple_passive_streams",kind='scatter', color='DarkRed', label='Passive', ax=ax) # av1.plot(y="total_rev",x="apple_collection_streams",kind='scatter', color='Yellow', label='Collection', ax=ax) # plt.xlabel('Apple Streams') # #print(plot4) #plot2, plot3, plot4, plot5, plot6) #print plt.hist(np.reciprocal(myds['TOTAL_GROSS_REVENUE']), bins=100) #print plt.hist(myds['TOTAL_GROSS_REVENUE'], bins=100,range=(10000,1000000)) fml = "total_rev ~ spotify_total_streams + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + apple_total_streams + apple_playlist_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "rev_log ~ spotify_total_streams + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + apple_total_streams + apple_playlist_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #formula for streaming: #fml = "apple_total_streams ~ apple_passive_percent + apple_playlist_percent + apple_active_percent + apple_collection_percent" #fml = "spotify_total_streams ~ spotify_passive_percent + spotify_playlist_percent + spotify_active_percent + spotify_collection_percent" #fml = "total_rev ~ spotify_total_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "total_rev ~ total_physical_rev + cd_units + num_active_releases + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" #fml = "rev_log ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_passive_streams + spotify_collection_streams + spotify_total_streams + apple_active_streams + apple_passive_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" result = sm.ols(fml, data=av).fit() print(result.summary()) print(df["total_rev"].mean()) #print(df.nlargest(100, 'spotify_playlist_percent')) plt.show()