{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "%run ./cluster_utils.ipynb\n", "%run ./plot_metrics.ipynb" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "from sklearn.cluster import DBSCAN\n", "from sklearn.preprocessing import StandardScaler\n", "import numpy as np\n", "import pandas as pd\n", "from sklearn import metrics\n", "import os\n", "\n", "\n", "import logging\n", "logging.getLogger().setLevel(logging.INFO)\n", "\n", "\n", "def get_best_epsilon(X, client_id):\n", " \"\"\" Get best epsilon value for clusters. This determine the nbr of clusters in DBSCAN algo. \"\"\"\n", " logging.info(\"Calculating epsilon for DBSCAN \")\n", " A = []\n", " B = []\n", " C = []\n", "\n", " # TODO! add mahalanobis back\n", " for i in np.linspace(0.1, 5, 50):\n", " db = DBSCAN(eps=i, min_samples=5, metric='euclidean').fit(X) # metric_params={'V': np.cov(X)}\n", "\n", " core_samples_mask = np.zeros_like(db.labels_, dtype=bool)\n", " core_samples_mask[db.core_sample_indices_] = True\n", " labels = db.labels_\n", " n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)\n", "\n", " sum = 0\n", " for t in labels:\n", " if t == -1:\n", " sum = sum + 1\n", " C.append(sum)\n", "\n", " A.append(i)\n", " B.append(int(n_clusters_))\n", "\n", " # TODO! automatically pick up best epsilon\n", "\n", " logging.info(f\"Distance: {A}\")\n", " results = pd.DataFrame([A, B, C]).T\n", " results.columns = ['distance', 'Number of clusters', 'Number of outliers']\n", " graph = results.plot(x='distance', y='Number of clusters', figsize=(10, 6))\n", " graph.legend()\n", "\n", " epsilon_graph = f'DBSCAN_epsilon_{client_id}.png'\n", " graph.figure.savefig(epsilon_graph)\n", "\n", " # plt.show()\n", " # TODO! Get best epsilon from results automatically\n", " return 0.5, epsilon_graph\n", "\n", "\n", "def run_dbscan_algo(client_id, dataset, cols, modelling_config):\n", "\n", " algo = 'DBSCAN'\n", " graph_locations = []\n", " X = dataset[cols]\n", " X2 = dataset[cols].copy()\n", "\n", " \"\"\" Decide where do we want to do standardscaling \"\"\"\n", " X = StandardScaler().fit_transform(X)\n", "\n", " # TODO! add silhouettes plots\n", "\n", " eps, epsilon_graph = get_best_epsilon(X, client_id)\n", " graph_locations.append(epsilon_graph)\n", "\n", " \"\"\" \n", " Mahalanobis Distance is used for calculating the distance between two data points in a multivariate space.\n", " \"\"\"\n", " # TODO! add Mahalanobis back 'metric_params': {'V': np.cov(X)}\n", " hyperparams = {'eps': eps, 'min_samples': 5, 'metric': 'euclidean'}\n", " print(\"fitting dbscan\")\n", " db = DBSCAN(**hyperparams).fit(X)\n", " core_samples_mask = np.zeros_like(db.labels_, dtype=bool)\n", " core_samples_mask[db.core_sample_indices_] = True\n", " labels = db.labels_\n", "\n", " dataset = dataset.join(pd.DataFrame(labels))\n", " dataset = dataset.rename(columns={0: 'cluster'})\n", "\n", " # TODO! can we do this more elegantly?\n", " # Create a dataframe of our standard-scaled stuff\n", " X = pd.DataFrame(X).join(pd.DataFrame(labels), lsuffix='left_')\n", " X = X.rename(columns={'0': 'cluster'})\n", "\n", " # X2 is already a dataframe\n", " X2 = X2.join(pd.DataFrame(labels), lsuffix='left_')\n", " X2 = X2.rename(columns={0: 'cluster'})\n", "\n", " # Number of clusters in labels, ignoring noise if present.\n", " n_clusters_ = len(set(labels)) - (1 if -1 in labels else 0)\n", " n_noise_ = list(labels).count(-1)\n", "\n", " logging.info('Estimated number of clusters: %d' % n_clusters_)\n", " logging.info('Estimated number of noise points: %d' % n_noise_)\n", " logging.info(\"Silhouette Coefficient: %0.3f\"\n", " % metrics.silhouette_score(X, labels))\n", "\n", " # TODO! color noise (label = -1) black or osmething distinct\n", "\n", " plot_2d = modelling_config.get('plot_2d')\n", " plot_3d = modelling_config.get('plot_3d')\n", " boxplot = modelling_config.get('boxplot')\n", " if boxplot:\n", " \"\"\" Plot boxplot \"\"\"\n", " #graph_boxplot_location = plot_boxplot(X, client_id, algo, 1)\n", " #graph_locations.append(graph_boxplot_location)\n", " \"\"\" Plot boxplot \"\"\"\n", " graph_boxplot_location = plot_boxplot(X2, client_id, algo, 2)\n", " graph_locations.append(graph_boxplot_location)\n", "\n", " if plot_2d:\n", " \"\"\" Plot 2d scatter version 1\"\"\"\n", " #graph_2d_v1_location = plot_2d_scatter(X, client_id, algo, 1)\n", " #graph_locations.append(graph_2d_v1_location)\n", " \"\"\" Plot 2d scatter version 2\"\"\"\n", " graph_2d_v2_location = plot_2d_scatter(X2, client_id, algo, 2)\n", " graph_locations.append(graph_2d_v2_location)\n", "\n", " if plot_3d:\n", " \"\"\" Plot 3d scatter version 1\"\"\"\n", " #graph_3d_v1_location = plot_3d_scatter(X, client_id, algo, 1)\n", " #graph_locations.append(graph_3d_v1_location)\n", " \"\"\" Plot 3d scatter version 2\"\"\"\n", " graph_3d_v2_location = plot_3d_scatter(X2, client_id, algo, 2)\n", " graph_locations.append(graph_3d_v2_location)\n", "\n", " eval_metrics = {'clusters': n_clusters_,\n", " 'noise': n_noise_,\n", " 'silhouette_coefficient': metrics.silhouette_score(X, labels)}\n", "\n", " output_cols = ['fan_id', 'cluster']\n", " cluster_output = dataset[output_cols]\n", " model = db\n", "\n", " return model, eval_metrics, hyperparams, cluster_output, graph_locations\n" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [] } ], "metadata": { "kernelspec": { "display_name": "conda_python3", "language": "python", "name": "conda_python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.6.5" } }, "nbformat": 4, "nbformat_minor": 4 }