import os import uuid from ast import literal_eval import matplotlib.cm as cm import matplotlib.pyplot as plt import numpy as np import pandas as pd import seaborn as sns from mpl_toolkits.mplot3d import Axes3D from sklearn.metrics import silhouette_samples, silhouette_score from sklearn.neighbors import NearestNeighbors from .. import mlflow_client from ..utils.aws_connectors import get_rds_engine, get_rendered_sql_template # Better aesthetics - https://seaborn.pydata.org/tutorial/aesthetics.html sns.set_style("whitegrid", {'axes.grid': False}) sns.set_color_codes('dark') sns.color_palette("Set2") sns.set_context("paper") sns.despine() import logging logging.getLogger().setLevel(logging.INFO) # TODO! Add all graphs plotting here # https://matplotlib.org/3.1.0/gallery/lines_bars_and_markers/scatter_with_legend.html # https://jovianlin.io/data-visualization-seaborn-part-3/ # https://matplotlib.org/3.1.1/gallery/lines_bars_and_markers/scatter_demo2.html#sphx-glr-gallery-lines-bars-and-markers-scatter-demo2-py def draw_graphs(training_set, labels, modelling_config, n_clusters, centers): # TODO! Refactor this so we wouldn't have to feed dataset and X separately, or other params plot_2d = modelling_config.get('plot_2d') plot_3d = modelling_config.get('plot_3d') boxplot = modelling_config.get('boxplot') plot_silhouette = modelling_config.get('plot_silhouette') graph_locations = [] if plot_2d or plot_3d or boxplot or plot_silhouette: X = pd.DataFrame(training_set).join(pd.DataFrame(labels), lsuffix='left_').rename(columns={0: 'cluster', '0': 'cluster'}) X = X.loc[:, ~X.columns.duplicated()] if boxplot: """ Plot boxplot """ try: graph_boxplot_location = plot_boxplot(X) graph_locations.append(graph_boxplot_location) except Exception as e: logging.error(f"Error when plotting boxplot: {e}") if plot_2d: """ Plot 2d scatter""" try: graph_2d_location = plot_2d_scatter(X) graph_locations.append(graph_2d_location) except Exception as e: logging.error(f"Error when plotting 2d scatter: {e}") if plot_3d: """ Plot 3d scatter version """ try: graph_3d_location = plot_3d_scatter(X) graph_locations.append(graph_3d_location) except Exception as e: logging.error(f"Error when plotting 3d scatter: {e}") if plot_silhouette: # Not all models allow plotting silhouette plots try: silhouette_locations_list = get_silhouette_plot(X, n_clusters, labels, centers) graph_locations.extend(silhouette_locations_list) except Exception as e: logging.error(f"Error when plotting silhouette: {e}") return graph_locations def draw_feature_importance(feature_importance, cols): """ Generate image from feature_importance""" importance_df = pd.DataFrame({'features': cols, 'importance': feature_importance}).sort_values(by='importance', ascending=False) fig, ax = plt.subplots() sns.barplot(x='importance', y='features', data=importance_df, label='Feature importance', # color='b', edgecolor='w' ) ax.set_yticklabels(importance_df['features'], size=5) current_path = os.path.dirname(__file__) unique_id = uuid.uuid4() feature_importance_plot_location = f'{current_path}/resources/feature_importance_{unique_id}.png' plt.savefig(feature_importance_plot_location, bbox_inches='tight') plt.close('all') return feature_importance_plot_location def draw_pairplot(feature_vector, importance_df, collection_id): """ To better show these variables in future plots, we can transform these columns by taking the logarithm of the values: # Take the log of population and gdp_per_capita df['log_pop'] = np.log10(df['pop']) https://towardsdatascience.com/visualizing-data-with-pair-plots-in-python-f228cf529166 https://seaborn.pydata.org/tutorial/axis_grids.html """ nbr_of_top_features = 5 importance_df = importance_df.sort_values(by='importance', ascending=False).head(nbr_of_top_features) #logging.info(importance_df) # TODO! handle typecasting more gracefully #logging.info(feature_vector[importance_df['features']].dtypes) """ Limit our feature_vector by taking only the top features from feature_importance. It also tries to convert fields to numeric where possible, which enables us plot better pairplots """ limited_feature_vector = feature_vector[importance_df['features']].apply(pd.to_numeric, errors='ignore') #limited_feature_vector = feature_vector[importance_df['features']].astype(float) # Get cluster from our original feature_vector limited_feature_vector['cluster'] = feature_vector['cluster'] print(f"Limited feature vector: {limited_feature_vector.columns}") current_path = os.path.dirname(__file__) plt.figure() sns.pairplot(limited_feature_vector, plot_kws={'alpha': 0.6, 's': 80, 'edgecolor': 'k'}, diag_kind='kde', hue='cluster' ) plot_location = f'{current_path}/collection_pairplot_{collection_id}.png' plt.savefig(plot_location, bbox_inches='tight') plt.close('all') def plot_boxplot(X): """ Plotting 2d scatterplot. We arbitrarily take first two cols. """ fig = plt.figure() g = sns.boxplot(x=X.columns.tolist()[1], y=X.columns.tolist()[0], #size=X.columns.tolist()[2], #hue='cluster', data=X) # TODO! get cluster value numbers, they are not shown atm g.legend(loc='best', bbox_to_anchor=(1, 1), ncol=1) plt.title(f"Estimated number of clusters: {len(set(X['cluster']))}") current_path = os.path.dirname(__file__) unique_id = uuid.uuid4() box_plot_location = f'{current_path}/resources/clusters_boxplot_{unique_id}.png' fig.savefig(box_plot_location, bbox_inches='tight') plt.close('all') return box_plot_location def plot_distance(X, algo): """ Plotting distance plot. https://towardsdatascience.com/machine-learning-clustering-dbscan-determine-the-optimal-value-for-epsilon-eps-python-example-3100091cfbc TODO! Similarily to get_best_elbow, get the curve and get best epsilon this way """ neigh = NearestNeighbors(n_neighbors=2) nbrs = neigh.fit(X) distances, indices = nbrs.kneighbors(X) distances = np.sort(distances, axis=0) distances = distances[:, 1] fig = plt.figure() plt.plot(distances) plt.title(f"Distance Plot") current_path = os.path.dirname(__file__) unique_id = uuid.uuid4() distance_plot_location = f'{current_path}/resources/{algo}_distance_plot_{unique_id}.png' fig.savefig(distance_plot_location, bbox_inches='tight') plt.close('all') return distance_plot_location def plot_2d_scatter(X): """ Plotting 2d scatterplot. We arbitrarily take first two cols. """ fig = plt.figure() g = sns.scatterplot(x=X.columns.tolist()[0], y=X.columns.tolist()[1], size=X.columns.tolist()[2], hue='cluster', palette=sns.color_palette('dark', n_colors=len(set(X['cluster']))), data=X) # TODO! get cluster value numbers, they are not shown atm g.legend(loc='best', bbox_to_anchor=(1.25, 1), ncol=1) plt.title(f"Estimated number of clusters: {len(set(X['cluster']))}") current_path = os.path.dirname(__file__) unique_id = uuid.uuid4() scatter_plot_location = f'{current_path}/resources/clusters_scatter_2d__{unique_id}.png' fig.savefig(scatter_plot_location, bbox_inches='tight') plt.close('all') return scatter_plot_location def plot_3d_scatter(X): # TODO! fix colors (they should come from seaborn, palette but are not """ We plot a 3d cluster map, because we almost always have more than 2 clusters. We arbitrarily take first 3 columns """ fig = plt.figure() ax = Axes3D(fig) xs = X[X.columns[0]] ys = X[X.columns[1]] zs = X[X.columns[2]] sns.set() #colors = cm.nipy_spectral(X['cluster'].astype(np.float) / n_clusters) g = ax.scatter(xs, ys, zs, s=50, alpha=0.6, edgecolors='grey', c=X['cluster'].astype(np.float), ) # TODO! why is this not showing ax.legend(*g.legend_elements(), loc="center left", title="Clusters") ax.set_xlabel(X.columns[0]) ax.set_ylabel(X.columns[1]) ax.set_zlabel(X.columns[2]) plt.title(f"Estimated number of clusters: {len(set(X['cluster']))}") current_path = os.path.dirname(__file__) unique_id = uuid.uuid4() scatter_plot_location = f'{current_path}/resources/clusters_scatter_3d_{unique_id}.png' fig.savefig(scatter_plot_location, bbox_inches='tight') plt.close('all') return scatter_plot_location def get_silhouette_plot(training_set, n_clusters, cluster_labels, centers): """ https://scikit-learn.org/stable/auto_examples/cluster/plot_kmeans_silhouette_analysis.html """ plot_locations = [] # Create a subplot with 1 row and 2 columns fig, (ax1, ax2) = plt.subplots(1, 2) fig.set_size_inches(18, 7) # The 1st subplot is the silhouette plot # The silhouette coefficient can range from -1, 1 but in this example all # lie within [-0.1, 1] ax1.set_xlim([-0.1, 1]) # The (n_clusters+1)*10 is for inserting blank space between silhouette # plots of individual clusters, to demarcate them clearly. ax1.set_ylim([0, len(training_set) + (n_clusters + 1) * 10]) # The silhouette_score gives the average value for all the samples. # This gives a perspective into the density and separation of the formed # clusters silhouette_avg = silhouette_score(training_set, cluster_labels) #print("For n_clusters =", n_clusters, # "The average silhouette_score is :", silhouette_avg) # Compute the silhouette scores for each sample sample_silhouette_values = silhouette_samples(training_set, cluster_labels) y_lower = 10 for i in range(n_clusters): # Aggregate the silhouette scores for samples belonging to # cluster i, and sort them ith_cluster_silhouette_values = \ sample_silhouette_values[cluster_labels == i] ith_cluster_silhouette_values.sort() size_cluster_i = ith_cluster_silhouette_values.shape[0] y_upper = y_lower + size_cluster_i color = cm.nipy_spectral(float(i) / n_clusters) ax1.fill_betweenx(np.arange(y_lower, y_upper), 0, ith_cluster_silhouette_values, facecolor=color, edgecolor=color, alpha=0.7) # Label the silhouette plots with their cluster numbers at the middle ax1.text(-0.05, y_lower + 0.5 * size_cluster_i, str(i)) # Compute the new y_lower for next plot y_lower = y_upper + 10 # 10 for the 0 samples ax1.set_title("The silhouette plot for the various clusters.") ax1.set_xlabel("The silhouette coefficient values") ax1.set_ylabel("Cluster label") # The vertical line for average silhouette score of all the values ax1.axvline(x=silhouette_avg, color="red", linestyle="--") ax1.set_yticks([]) # Clear the yaxis labels / ticks ax1.set_xticks([-0.1, 0, 0.2, 0.4, 0.6, 0.8, 1]) # 2nd Plot showing the actual clusters formed colors = cm.nipy_spectral(cluster_labels.astype(float) / n_clusters) ax2.scatter(training_set.iloc[:, 0], training_set.iloc[:, 1], marker='.', s=30, lw=0, alpha=0.7, c=colors, edgecolor='k') """ This is not possible for DBSCAN and Spectral (because we don't have centers.. although we may calculate "mean", but of arbitrary data shape and clusters, this "center" might be meaningless still. """ try: # Draw white circles at cluster centers ax2.scatter(centers[:, 0], centers[:, 1], marker='o', c="white", alpha=1, s=200, edgecolor='k') for i, c in enumerate(centers): ax2.scatter(c[0], c[1], marker='$%d$' % i, alpha=1, s=50, edgecolor='k') except Exception as e: logging.error(e) ax2.set_title("The visualization of the clustered data.") ax2.set_xlabel("Feature space for the 1st feature") ax2.set_ylabel("Feature space for the 2nd feature") plt.suptitle((f"Silhouette analysis clustering on sample data " "with n_clusters = %d" % n_clusters), fontsize=14, fontweight='bold') current_path = os.path.dirname(__file__) # TODO! For resiliency, it should be some unique filename unique_id = uuid.uuid4() silhouette_plot_location = f'{current_path}/resources/silhouette_n_{n_clusters}_{unique_id}.png' fig.savefig(silhouette_plot_location, bbox_inches='tight') plot_locations.append(silhouette_plot_location) plt.close('all') return plot_locations def get_mlflow_run(client_id, algo, collection_id, run_id=None): """ :param client_id: hashed client identifier :param collection_id: :return: run info dataframe """ experiment = mlflow_client.get_experiment_by_name(client_id) all_run_infos = mlflow_client.list_run_infos(experiment.experiment_id) feature_importance_df = None for run in all_run_infos: if run.status != 'FINISHED': continue """ Sometimes we know exactly what is the run_id we can't to analyse """ if run_id: run_id_filter = run_id else: run_id_filter = run.run_id full_run = mlflow_client.get_run(run_id_filter).to_dictionary() #logging.info('Got run') """ Only filter for specific modelling runs by a special tag""" tags = full_run['data'].get('tags') if not tags.get('modelling_type') == 'Data modelling': continue params = full_run['data'].get('params') if not str(params.get('collection_id')) == str(collection_id): continue metrics = full_run['data'].get('metrics') # TODO! get good silhouette score silhouette_coefficient = metrics.get('silhouette_coefficient') """ TODO! Just pick the first model that is with our algo name """ if algo in tags.get('model_name').lower(): importance = literal_eval(params.get('feature_importance')) features = literal_eval(params.get('original_features')) feature_importance_df = pd.DataFrame({'features': features, 'importance': importance}) model_name = tags.get('model_name') break logging.info(f"Got model {model_name} with silhouette_coefficient {silhouette_coefficient}") return feature_importance_df if __name__ == '__main__': client_id = 'c7b538defdde08de612a48a0b9a7a1725ace9bfa49d74a7909d81e856' parent_collection_id = 35 algo = 'kmeans' # TODO! how to pick best model? run_id = 'b63be3b91b504490818a201d7b5917db' feature_importance_df = get_mlflow_run(client_id, algo, parent_collection_id, run_id) DATABASE_NAME = 'fansifter-dev' engine = get_rds_engine(DATABASE_NAME) """ Get processed and feature engineered collection id, We can't do RFM on raw dataset (unless we have labelled data from frontend) """ sql = f"""SELECT max(id) as id FROM {client_id}.collection WHERE parent_id = {parent_collection_id} AND source = 'feature_engineering' """ try: collection_id = pd.read_sql(sql, engine).iloc[0].values[0] except IndexError as e: logging.error(f"Could not find processed and feature engineered collection for parent id {parent_collection_id}: {e}") feature_vector_params = {'schema_name': client_id, 'collection_id': collection_id, 'parent_collection_id': parent_collection_id, } feature_vector = "get_collection_feature_engineered.sql" sql = get_rendered_sql_template(feature_vector_params, feature_vector) feature_vector = pd.read_sql_query(sql, engine) # TODO! get cluster from s3 instead sql = f"SELECT * FROM {client_id}.collection_{parent_collection_id}_{algo};" cluster_vector = pd.read_sql_query(sql, engine) feature_vector['cluster'] = cluster_vector['cluster'] draw_pairplot(feature_vector, feature_importance_df, collection_id) # python3 -m service.tasks.plot_metrics