import logging import os import uuid import hdbscan import numpy as np import pandas as pd from sklearn import metrics from sklearn.cluster import DBSCAN from ..plot_metrics import draw_graphs # from cdbw import CDbw logging.getLogger().setLevel(logging.INFO) # TODO! Optimize this somehow. Unfortunately dask-ml does not have DBSCAN. Neither does using dask parallel backend work well with multiprocessing. # https://github.com/dask/dask-tutorial/issues/80 def get_best_epsilon(X, nr_of_features): """ Get best epsilon value for clusters. This determine the nbr of clusters in DBSCAN algo. """ logging.info("Calculating epsilon for DBSCAN ") A = [] B = [] C = [] D = [] E = [] # TODO! add mahalanobis back for j in [5, nr_of_features]: # 5 is the default one provided by algorithm for i in np.linspace(0.1, 1, 10): db = DBSCAN(eps=i, min_samples=j, metric='euclidean').fit(X) # metric_params={'V': np.cov(X)} core_samples_mask = np.zeros_like(db.labels_, dtype=bool) core_samples_mask[db.core_sample_indices_] = True labels = db.labels_ n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) try: s_score = metrics.silhouette_score(X, labels) except Exception as e: logging.warning(f"Cannot calculate silhouette score for algo DBSCAN: {e}") s_score = 0 sum = 0 for t in labels: if t == -1: sum = sum + 1 C.append(sum) A.append(i) B.append(int(n_clusters_)) D.append(s_score) E.append(j) # TODO! automatically pick up best epsilon # logging.info(f"Distance: {A}") results = pd.DataFrame([A, B, C, D, E]).T results.columns = ['distance', 'Number of clusters', 'Number of outliers', 'silhouette score', 'nr of features'] results.sort_values(['silhouette score', 'distance'], ascending=[False, True], inplace=True) new_eps = results['distance'].iloc[0] # pick the eps for run with highest silhouette score new_feature_nrs = results['nr of features'].iloc[0] # pick the min_samples parameter for run with highest silhouette score graph = results.plot(x='distance', y='Number of clusters', figsize=(10, 6)) graph.legend() current_path = os.path.dirname(__file__) unique_id = uuid.uuid4() epsilon_graph = f'{current_path}/../resources/DBSCAN_epsilon_{unique_id}.png' graph.figure.savefig(epsilon_graph) # plt.show() # TODO! Get best epsilon from results automatically return new_eps, new_feature_nrs, epsilon_graph def run_algo(training_set, fan_ids, modelling_config, algo_params): X = training_set new_min_samples = X.shape[1] * 2 # one of the rule of thumb approach if isinstance(fan_ids, pd.Series): fan_ids = fan_ids.to_frame() best_eps, best_min_samples, epsilon_graph = get_best_epsilon(X, new_min_samples) # graph_locations.append(epsilon_graph) hyperparams = algo_params['hyperparams'] # eps = algo_params['eps'] hyperparams['eps'] = best_eps hyperparams['min_samples'] = best_min_samples # hyperparams['random_state'] = 0 # This algo doesn't have random_state hyperparams['n_jobs'] = -1 # We always want to use all CPU cores """ Mahalanobis Distance is used for calculating the distance between two data points in a multivariate space. """ db = DBSCAN(**hyperparams).fit(X) labels = db.labels_ validity_index = hdbscan.validity.validity_index(X, labels, metric="euclidean") # core_samples_mask = np.zeros_like(labels, dtype=bool) # core_samples_mask[db.core_sample_indices_] = True cluster_output = fan_ids.join(pd.DataFrame(labels)).rename(columns={0: 'cluster', '0': 'cluster'}) X = pd.DataFrame(X).join(pd.DataFrame(labels), lsuffix='left_').rename(columns={0: 'cluster', '0': 'cluster'}) # Number of clusters in labels, ignoring noise if present. n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0) n_noise_ = list(labels).count(-1) # TODO! color noise (label = -1) black or osmething distinct # DBSCAN doesn't have centers graph_locations = draw_graphs(X, labels, modelling_config, n_clusters_, None) """ Gets S_DBW score. According to the following paper S_Dbw performs best across 5 different aspects of clustering when compared against Silhouette, DB, Dunn, Calinski, etc. http://datamining.rutgers.edu/publication/internalmeasures.pdf """ # s_dbw = CDbw(X, labels, metric="euclidean", alg_noise='comb', # intra_dens_inf=False, s=3, multipliers=False) try: s_score = metrics.silhouette_score(X, labels) except Exception as e: logging.warning(f"Cannot calculate silhouette score for algo DBSCAN: {e}") s_score = 0 # silhouette_score is looked at when choosing best model eval_metrics = {'clusters': n_clusters_, 'noise': n_noise_, 'silhouette_coefficient': s_score, 'validity_index': validity_index # 's_dbw': s_dbw } return db, eval_metrics, hyperparams, cluster_output, graph_locations if __name__ == '__main__': """ Instead of generating df, perhaps load your own dataframe from csv. We need a dataframe with training cols only (preprocessed, and standardscaled) and a fan_id. """ training_set = pd.DataFrame({ 'fan_id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19], 'a': [1, 2, 2, 4, 2, 6, 4, 3, 1, 5, 5, 6, 7, 9, 10, 11, 12, 13, 14], 'b': [2, 3, 3, 3, 3, 7, 8, 9, 2, 3, 4, 5, 3, 9, 10, 11, 12, 13, 14] }) print(training_set) fan_ids = training_set['fan_id'] modelling_config = { 'plot_2d': True, 'plot_3d': True, 'plot_silhouette': True, 'boxplot': True, } algo_params = {'eps': 0.1, 'hyperparams': {'min_samples': 5, 'metric': 'euclidean'}} print(run_algo(training_set, fan_ids, modelling_config, algo_params)[1])