#POLYNOMIAL FEATURES HERE import numpy as np import pandas as pd import seaborn as sns import statsmodels.formula.api as sm import statsmodels.imputation.mice as mice from statsmodels.regression.linear_model import OLS from collections import defaultdict import matplotlib.pyplot as plt from sklearn.feature_selection import VarianceThreshold from sklearn.preprocessing import PolynomialFeatures from sklearn.feature_selection import SelectKBest from sklearn.feature_selection import chi2 #http://www.dummies.com/programming/big-data/data-science/data-science-how-to-create-interactions-between-variables-with-python/ from sklearn.linear_model import LinearRegression from sklearn.cross_validation import cross_val_score from sklearn.cross_validation import KFold regression = LinearRegression(normalize=True) myds = pd.read_csv('catalog_result.csv') #m=2373 #myds = pd.read_csv('catalog_result_socials.csv', encoding = "ISO-8859-1") #m=1093 #socials = pd.read_csv('social_result.csv') #social data by id only genres = pd.get_dummies(myds['RELEASE_GENRE']) LABEL_COUNTRY = pd.get_dummies(myds['LABEL_COUNTRY']) df = pd.DataFrame({ "total_rev": myds['TOTAL_GROSS_REVENUE'], "rev_recip": np.reciprocal(myds['TOTAL_GROSS_REVENUE']), "rev_log": np.log(myds['TOTAL_GROSS_REVENUE']), "total_physical_rev": myds['PHYSICAL_GROSS_REVENUE'], "num_cds": myds['NUM_OF_CDS'], "num_active_releases": myds['NUM_ACTIVE_RELEASES'], "cd_units": myds['CD_REVENUE_UNITS_SOLD'], "spotify_passive_streams": myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL'], "spotify_playlist_streams": myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL'], "spotify_active_streams": myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL'], "spotify_collection_streams": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL'], "spotify_total_streams": myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_passive_percent": myds['SPOTIFY_STREAMS_PASSIVE_DISC_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_playlist_percent": myds['SPOTIFY_PLAYLISTS_STREAMS_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_active_percent": myds['SPOTIFY_STREAMS_ACTIVE_DISC_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "spotify_collection_percent": myds['SPOTIFY_STREAMS_COLLECTION_TOTAL']/myds['SPOTIFY_OVERALL_STREAMS_TOTAL'], "apple_passive_streams": myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL'], "apple_playlist_streams": myds['APPLE_PLAYLISTS_STREAMS_TOTAL'], "apple_active_streams": myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL'], "apple_collection_streams": myds['APPLE_STREAMS_COLLECTION_TOTAL'], "apple_total_streams": myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_passive_percent": myds['APPLE_STREAMS_PASSIVE_DISC_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_playlist_percent": myds['APPLE_PLAYLISTS_STREAMS_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_active_percent": myds['APPLE_STREAMS_ACTIVE_DISC_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "apple_collection_percent": myds['APPLE_STREAMS_COLLECTION_TOTAL']/myds['APPLE_OVERALL_STREAMS_TOTAL'], "amazon_total_streams": myds['AMAZON_OVERALL_STREAMS_TOTAL'], "pandora_total_streams": myds['PANDORA_OVERALL_STREAMS_TOTAL'], "youtube_audio_streams": myds['YOUTUBE_AUDIO_OVERALL_STREAMS_TOTAL'], "shazam_total_streams": myds['SHAZAM_OVERALL_STREAMS_TOTAL'], "alexa_total_streams": myds['ALEXA_OVERALL_STREAMS_TOTAL'], "itunes_total_downloads": myds['ITUNES_TOTAL_DOWNLOADS'], "amazon_total_downloads": myds['AMAZON_TOTAL_DOWNLOADS'], "google_total_downloads": myds['GOOGLE_PLAY_DOWNLOADS_TOTAL'], }) df_soc = pd.DataFrame({ "fb_likes": myds['FB_LIKES_LTD'], "twitter_followers": myds['TWITTER_FOLLOWERS_LTD'], "youtube_subs": myds['YOUTUBE_SUBS_LTD'], #"wiki_pageviews": myds['WIKI_PAGEVIEWS_LTD'], "insta_followers": myds['INSTA_FOLLOWERS_LTD'], #"spotify_followers": myds['SPOTIFY_FOLLOWERS_LTD'], "fb_talking_about": myds['FB_TALKING_ABOUT_TD'], #"vevo_views": myds['VEVO_VIDEO_VIEWS_LTD'], "twitter_mentions": myds['TWITTER_MENTIONS_TD'], "twitter_retweets": myds['TWITTER_RETWEETS_TD'], "youtube_likes": myds['YOUTUBE_LIKES_LTD'], "insta_comments": myds['INSTAGRAM_COMMENTS_LTD'], "songkick_followers": myds['SONGKICK_FOLLOWERS_LTD'], }) #impute NaN with 0s df = df.fillna(0) genres = genres.fillna(0) LABEL_COUNTRY = LABEL_COUNTRY.fillna(0) #set negative values to 0 df[df < 0] = 0 print("df size:") print(df.shape) df_soc[df_soc < 0] = 0 #create full all_vars table all_vars = pd.concat([df.fillna(0), genres, LABEL_COUNTRY, df_soc] , axis=1) print("shape with social NaNs:") print(all_vars.shape) #egt rid of NaNs in socials all_vars = all_vars.dropna() print("shape with dropped NaNs:") print(all_vars.shape) # create interaction features # poly_vars = PolynomialFeatures(interaction_only=True) # all_vars = poly_vars.fit_transform(all_vars, y=all_vars["total_rev"]) # print("shape with all interactions:") # print(all_vars.shape) # print("feature names:") # print(list(all_vars)) #http://www.dummies.com/programming/big-data/data-science/data-science-how-to-create-interactions-between-variables-with-python/ #crossvalidation = KFold(n=all_vars.shape[0], n_folds=10, shuffle=True, random_state=1) interactions = list() for feature_A in list(all_vars): for feature_B in list(all_vars): if feature_A not in ['total_rev','rev_recip','rev_log'] and feature_B not in ['total_rev','rev_recip','rev_log']: if feature_A > feature_B: all_vars['interaction'] = all_vars[feature_A] * all_vars[feature_B] cc = np.corrcoef(all_vars['interaction'],y=all_vars['total_rev'])[0][1] if not np.isnan(cc): interactions.append((feature_A, feature_B, cc)) interactions.sort(key=lambda x: abs(x[2]), reverse=True) for elem in interactions: print(elem) # remove all features whose variance < 0.6 # sel = VarianceThreshold(threshold=.6) # all_vars = sel.fit_transform(all_vars) # print("shape after thresholding:") # print(all_vars.shape) #select 20 best features #all_vars_select = SelectKBest(chi2, k=20).fit_transform(all_vars[:,3:], all_vars[:,0:1]) #print list(all_vars_select) # fml = "total_rev ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" # #fml = "rev_recip ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" # #fml = "rev_log ~ total_physical_rev + num_cds + num_active_releases + cd_revenue + spotify_playlist_streams + spotify_active_streams + spotify_collection_streams + spotify_total_streams + apple_playlist_streams + apple_active_streams + apple_collection_streams + apple_total_streams + amazon_total_streams + pandora_total_streams + shazam_total_streams + alexa_total_streams + youtube_audio_streams + itunes_total_downloads + amazon_total_downloads + google_total_downloads" # corr2 = np.corrcoef(all_vars_select) # print(corr2) # print(sns.heatmap(corr2)) # result = sm.ols(fml, data=all_vars).fit() # print result.summary() #print result.summary() #print plt.hist(np.reciprocal(myds['TOTAL_GROSS_REVENUE']), bins=100) #print plt.hist(myds['TOTAL_GROSS_REVENUE'], bins=100,range=(10000,1000000)) plt.show()