import logging import pandas as pd import numpy as np import matplotlib.pyplot as plt from sklearn.feature_selection import RFECV from sklearn.metrics import classification_report, confusion_matrix, accuracy_score, log_loss from sklearn.model_selection import GridSearchCV from sklearn.preprocessing import MinMaxScaler, StandardScaler from sklearn.linear_model import LogisticRegression from sklearn.ensemble import RandomForestClassifier from sklearn.inspection import permutation_importance from sklearn.utils import class_weight from abc import ABC, abstractmethod class Model(ABC): def __init__(self): super().__init__() logging.info('Initialising Model') @abstractmethod def train(self): logging.info('Training Model') @abstractmethod def predict(self): logging.info('Predicting...') ############################################################################################################# ####################### Logistic Regression ################################################################ ############################################################################################################# class LogisticRegressionModel(Model): def __init__(self, prepared_data, outcome, cutoff = 0.50): super().__init__() self.X = prepared_data self.y = outcome self.model = None self.cutoff = cutoff self.class_weights = None def train(self): best_params = self.search_best_parameters(self.X, self.y) self.class_weights = class_weight.compute_class_weight('balanced', classes=np.unique(self.y), y=self.y.values.reshape(-1)) self.model = LogisticRegression(solver = best_params['solver'], C = best_params['C'], max_iter = best_params['max_iter'], penalty = best_params['penalty'], class_weight = 'balanced', n_jobs = -1).fit(self.X, self.y.values.ravel()) #class_weight = dict(enumerate(self.class_weights))).fit(self.X, self.y.values.ravel()) # <--------------- tried this for cost-sensitive learning (by assigning higher misclassification cost to the minority class, model is encouraged to prioritise correctly the identification of this class. def predicted_probabilities(self, X_to_predict): return self.model.predict_proba(X_to_predict) def predict(self, X_to_predict): pred = self.predicted_probabilities(X_to_predict) prob_1 = [x[1] for x in pred] return np.where(np.array(prob_1) > self.cutoff, 1, 0) def model_intercept(self): #print(self.model.estimator_.intercept_) print(self.model.intercept_) def model_coefficients(self): for coefficients, feature in zip(self.model.coef_[0], self.X.columns): print(f"{feature}: {coefficients}") def search_best_parameters(self, X_train, y_train): """ Using grid search to identify a good set of parameters """ logging.info('Searching params for Logistic Regression ... \n') param_grid = { #'solver': ['liblinear', 'newton-cg', 'lbfgs', 'saga'], # solver and penalty params can't be on at the same time. l1 and l2 support different solvers 'solver': ['liblinear', 'saga'], 'C': [0.001, 0.01, 0.1, 1, 10, 100, 1000], 'max_iter': [5000], 'penalty' : ['l1', 'l2']} # CV_logreg = GridSearchCV(estimator=LogisticRegression(random_state=2023), param_grid=param_grid, cv=10, error_score='raise', n_jobs = -1) CV_logreg.fit(X_train, y_train.values.ravel()) result = CV_logreg.best_params_ logging.info('Best parameters found: %s', result) return result def important_features(self): model_fi = permutation_importance(self.model, self.X, self.y, scoring = 'f1') importances = model_fi.importances_mean for feature, importance in zip(self.X.columns, importances): print(f"{feature}: {importance}") def ROC_plot(self): return sklearn.metrics.plot_roc_curve(CV_logreg, X_train, y_train.values.ravel()) def estimate_predictor(model, X_test, y_test): """ Estimates accuracy of model """ pred_y = model.predict(X_test) observed_y = y_test return classification_report(observed_y, pred_y, output_dict=True) def confusion_matrix_calculation(model, X_test, y_test): pred_y = model.predict(X_test) observed_y = y_test return confusion_matrix(observed_y, pred_y) def accuracy_calculator(model, X_test, y_test): pred_y = model.predict(X_test) observed_y = y_test return accuracy_score(observed_y, pred_y) def loss_evaluation(model, X_test, y_test): pred_y = model.predict(X_test) observed_y = y_test return log_loss(observed_y, pred_y)