import logging from sklearn import metrics from sklearn_extra.cluster import KMedoids from ..plot_metrics import draw_graphs logging.getLogger().setLevel(logging.INFO) import pandas as pd import hdbscan def run_algo(training_set, fan_ids, modelling_config, algo_params): X = training_set if isinstance(fan_ids, pd.Series): fan_ids = fan_ids.to_frame() """" 'build' init is very non-robust if there are outliers in the dataset, and we're forced to use 'build' so we can use max_iter 0 (which helps us to train on bigger datasets). - https://github.com/scikit-learn-contrib/scikit-learn-extra/pull/73 So therefor, remove outliers. NB! This removes extreme outliers, but TODO! test hwhich is good outlier removal """ """ In case you need to remove outliers, use this X = X[X.apply(lambda x: np.abs(x - x.mean()) / x.std() < 10).all(axis=1)] """ hyperparams = algo_params['hyperparams'] n_clusters = algo_params['n_clusters'] hyperparams['n_clusters'] = n_clusters hyperparams['random_state'] = 0 # For consistency over multiple runs, we want this to always remain the same #hyperparams['n_jobs'] = -1 # We don't have n_jobs for this algo """ As per docs, max_iter can be zero in which case only the initialization is computed which may be suitable for large datasets when the initialization is sufficiently efficient (i.e. for 'build' init)""" kmediods = KMedoids(**hyperparams).fit(X) labels = kmediods.labels_ centers = kmediods.cluster_centers_ validity_index = hdbscan.validity.validity_index(X, labels, metric="euclidean") cluster_output = fan_ids.join(pd.DataFrame(labels)).rename(columns={0: 'cluster', '0': 'cluster'}) X = pd.DataFrame(X).join(pd.DataFrame(labels), lsuffix='left_').rename(columns={0: 'cluster', '0': 'cluster'}) graph_locations = draw_graphs(X, labels, modelling_config, n_clusters, centers) eval_metrics = {'clusters': n_clusters, 'intertia': kmediods.inertia_, 'silhouette_coefficient': metrics.silhouette_score(X, labels), 'davies_bouldin_index': metrics.davies_bouldin_score(X, labels), 'calinski_harabasz_index': metrics.calinski_harabasz_score(X, labels), 'validity_index': validity_index } return kmediods, eval_metrics, hyperparams, cluster_output, graph_locations if __name__ == '__main__': """ Instead of generating df, perhaps load your own dataframe from csv. We need a dataframe with training cols only (preprocessed, and standardscaled) and a fan_id. """ training_set = pd.DataFrame({ 'fan_id': [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14], 'a': [1,2,3,4,5,6,4,3,1,5,5,6,7,9], 'b': [2,3,4,5,6,7,8,9,2,3,4,5,6,9] }) fan_ids = training_set['fan_id'] modelling_config = { 'plot_2d': True, 'plot_3d': True, 'plot_silhouette': True, 'boxplot': True, } algo_params = {'n_clusters': 4, 'hyperparams': {'init': 'k-medoids++', 'max_iter': 0, 'method': 'pam', 'metric': 'euclidean'}} print(run_algo(training_set, fan_ids, modelling_config, algo_params)[1])