0.0.1b6 released with basic scikit-learn helper functions

Author: ngupta23

README.md

@@ -4,14 +4,14 @@ This is a helper package for a variety of functions as described in the Overview
 
 # Installation
 
-pip install more==0.0.1b5
+pip install more==0.0.1b6
 
 # Overview
 
 This is a helper package for a variety of functions
 1. Extension for Pandas Dataframe (Beta version released)
 2. Extension for Visualization (Beta version released)
-3. Extension for Scikit-learn (TBD)
+3. Extension for Scikit-learn (Beta version released)
 
 # Examples
 Check out the  [examples](https://github.com/ngupta23/more/tree/master/examples) folder for details on usage

build/lib/more/scikit_helper/__init__.py

@@ -1 +1,4 @@
-name = "Scikit Helper"
+from .plot_learning_curves import plot_learning_curves, plot_learning_curve
+from .plot_hyper_validation_curves import plot_hyper_validation_curves
+from .train_classifier import train_classifier
+from .plot_classification import print_classification_details, plot_classification_report

build/lib/more/scikit_helper/common.py

@@ -0,0 +1,17 @@
+# For Time Logging
+import time
+from contextlib import contextmanager
+import logging
+
+@contextmanager
+# Timing Function
+def time_usage(name=""):
+    """
+    log the time usage in a code block
+    """
+    #print ("In time_usage runID = {}".format(runID))
+    start = time.time()
+    yield
+    end = time.time()
+    elapsed_seconds = float("%.10f" % (end - start))
+    logging.info('%s: Time Taken (seconds): %s', name, elapsed_seconds)

build/lib/more/scikit_helper/plot_classification.py

@@ -0,0 +1,52 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+
+from sklearn import metrics as mt
+
+
+def print_classification_details(actual,predicted,verbose=False):
+    # print the accuracy and confusion matrix 
+    cm = mt.confusion_matrix(actual,predicted)
+    cr = mt.classification_report(actual,predicted)
+    
+    print("confusion matrix\n", cm)
+    print(cr)
+    
+    if (verbose == True):
+        plot_classification_report(cr)
+    
+    
+def plot_classification_report(cr, title=None, cmap='RdBu'):
+    """
+    Adapted from https://medium.com/district-data-labs/visual-diagnostics-for-more-informed-machine-learning-7ec92960c96b
+    """
+    title = title or 'Classification report'
+    lines = cr.split('\n')
+    classes = []
+    matrix = []
+  
+    for line in lines[2:(len(lines)-5)]:
+        s = line.split()
+        classes.append(s[0])
+        value = [float(x) for x in s[1: len(s) - 1]]
+        matrix.append(value)
+
+    fig, ax = plt.subplots(1)
+
+    for column in range(len(matrix)+1):
+        for row in range(len(classes)):
+            txt = matrix[row][column]
+            #ax.text(column,row,matrix[row][column],va='center',ha='center')
+            ax.text(column,row,txt,va='center',ha='center',size="x-large",bbox=dict(facecolor='white', alpha=0.5))
+
+    fig = plt.imshow(matrix, interpolation='nearest', cmap=cmap, vmin=0, vmax=1)
+    plt.title(title)
+    plt.colorbar()
+    x_tick_marks = np.arange(len(classes)+1)
+    y_tick_marks = np.arange(len(classes))
+    plt.xticks(x_tick_marks, ['Precision', 'Recall', 'F1-score'], rotation=45)
+    plt.yticks(y_tick_marks, classes)
+    plt.ylabel('Classes')
+    plt.xlabel('Measures')
+    plt.show()

build/lib/more/scikit_helper/plot_hyper_validation_curves.py

@@ -0,0 +1,152 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+
+import math
+
+from sklearn.model_selection import GridSearchCV
+
+
+##################################### 
+#### Grid Search Hyperparameters ####
+#####################################
+
+
+# Not loaded yet
+# If you want to release this, include this in the __init__.py file, and show an example of how to use.
+
+def plot_hyper_validation_curve(estimator, X, y, param_name, param_range
+                                ,title = "Validation Curve", xlabel = None, legend_loc='best', logX=False
+                                ,cv=None,scoring="accuracy",n_jobs=None,verbose=0, ax = None):
+    """
+    * Adapted from: 
+        - https://chrisalbon.com/machine_learning/model_evaluation/plot_the_validation_curve/
+        - https://scikit-learn.org/stable/auto_examples/model_selection/plot_validation_curve.html#sphx-glr-auto-examples-model-selection-plot-validation-curve-py
+    
+    """
+      
+    # Calculate accuracy on training and test set using range of parameter values
+    train_scores, test_scores = validation_curve(estimator=estimator 
+                                                 ,X=X,y=y 
+                                                 ,param_name=param_name 
+                                                 ,param_range=param_range
+                                                 ,cv=cv 
+                                                 ,scoring=scoring
+                                                 ,verbose=verbose
+                                                 ,n_jobs=n_jobs
+                                                )
+    
+
+    # Calculate mean and standard deviation for training set scores
+    train_mean = np.mean(train_scores, axis=1)
+    train_std = np.std(train_scores, axis=1)
+
+    # Calculate mean and standard deviation for test set scores
+    test_mean = np.mean(test_scores, axis=1)
+    test_std = np.std(test_scores, axis=1)
+    
+    # Clean up for log X axis plotting
+    x_label_append = "" # empty if not log scale
+    if (logX == True):
+        param_range = np.log10(param_range)
+        x_label_append = " (log scale)"
+    
+    # Plot mean accuracy scores for training and test sets
+    ax.plot(param_range, train_mean, label="Training Score", color="r")
+    ax.plot(param_range, test_mean, label="Cross-Validation Score", color="g")
+
+    # Plot accurancy bands for training and test sets
+    ax.fill_between(param_range, train_mean - train_std, train_mean + train_std, color="r",alpha=0.1)
+    ax.fill_between(param_range, test_mean - test_std, test_mean + test_std, color="g",alpha=0.1)
+
+    # Add annotations 
+    ax.title.set_text(title)
+    ax.set_ylabel((scoring + " Score").title())
+    if xlabel == None:
+        ax.set_xlabel("Hyperparameter: " + param_name + x_label_append)
+    else:
+        ax.set_xlabel(xlabel + x_label_append)
+    
+    ax.legend(loc=legend_loc)
+
+def plot_hyper_validation_curves(estimator, X, y, param_grid, scoring
+                                  ,logX=False
+                                  ,cv=None, refit = 'Accuracy'
+                                  ,n_jobs=None,verbose=0,arFigsize=None ):
+    
+    """
+    * Adapted from https://scikit-learn.org/stable/auto_examples/model_selection/plot_multi_metric_evaluation.html
+    * Uses GridSearchCV to evaluate many metrics at once. Hence, if evaluating many metrics at once, 
+      this is often faster than using validation_curve which will have to be run (trained) once time for each metric)
+      
+    * Currently supports only one hyperparameter at a time, but can score multiple metrics against that hyperparameter.
+    """
+    
+    gs = GridSearchCV(estimator = estimator
+                      ,param_grid=param_grid
+                      ,scoring=scoring, cv=cv, refit=refit
+                      ,return_train_score=True,n_jobs=-1,verbose=verbose)
+    gs.fit(X, y)
+    results = gs.cv_results_ 
+    
+    # Plotting the result
+    
+    num_rows = math.ceil(len(scoring)/2)
+    if (arFigsize == None):
+        arFigsize = (12,num_rows*4)
+    fig, axes = plt.subplots(num_rows, 2, figsize=arFigsize,squeeze =False)
+       
+    # For this function, we only expect 1 key
+    # If you have more than 1 parameter for GC, then results may not be as expected
+    for hyper in param_grid:
+        # Get the regular numpy array from the MaskedArray
+        X_axis = np.array(results['param_' + hyper].data, dtype=float)    
+        
+        x_label_append = " "
+        if (logX == True):
+            X_axis = np.log10(X_axis)
+            x_label_append = " (log scale)"
+
+        i = 0
+        for scorer in scoring:
+            title = "Validation Curve: " + scorer.title()
+                       
+            ax=axes[math.floor(i/2),i%2]
+            for sample, style, color in (('train', 'o-', 'r'), ('test', 'o-', 'g')):
+                sample_score_mean = results['mean_%s_%s' % (sample, scorer)]
+                sample_score_std = results['std_%s_%s' % (sample, scorer)]
+
+                ax.plot(X_axis, sample_score_mean, style, color=color,
+                        #alpha=1 if sample == 'test' else 1,
+                        label="%s (%s)" % (scorer, sample))
+
+                ax.fill_between(X_axis, sample_score_mean - sample_score_std,
+                                sample_score_mean + sample_score_std,
+                                alpha=0.1 if sample == 'test' else 0.1, color=color)
+
+                # Add annotations 
+                ax.title.set_text(title)
+                ax.set_ylim(-0.1, 1.1)
+                ax.set_ylabel(scorer.title() + " Score")
+                ax.set_xlabel("Hyperparameter: " + hyper + x_label_append)
+                ax.grid()
+                ax.legend(loc="best")
+
+            best_index = np.nonzero(results['rank_test_%s' % scorer] == 1)[0][0]
+            best_score = results['mean_test_%s' % scorer][best_index]
+
+            # Plot a dotted vertical line at the best score for that scorer marked by x
+            ax.plot([X_axis[best_index], ] * 2, [0, best_score],
+                    linestyle='-.', color='black', marker='x', markeredgewidth=3, ms=8)
+
+            # Annotate the best score for that scorer
+            ax.annotate("%0.2f" % best_score,
+                        (X_axis[best_index], best_score + 0.005))
+
+
+            i = i+1 # Increment for each scorer
+    #return(fig)    
+    
+
+

build/lib/more/scikit_helper/plot_learning_curves.py

@@ -0,0 +1,133 @@
+import numpy as np
+import pandas as pd
+import matplotlib.pyplot as plt
+import seaborn as sns
+import math
+
+from sklearn.model_selection import learning_curve
+from .common import time_usage
+
+#########################
+#### Learning Curves ####
+#########################
+
+
+def plot_learning_curve(estimator, title_suffix, X, y, scoring="accuracy", ylim=None, cv=None,
+                         n_jobs=None, train_sizes=np.linspace(.2, 1.0, 5),verbose=1, ax = None):
+    """
+    NOTE: Adopted from: https://scikit-learn.org/stable/auto_examples/model_selection/plot_learning_curve.html,
+    but allows plotting multiple metrics at a time when called from within plot_learning_curves (note the s at the end)
+    
+    Generate a simple plot of the test and training learning curve.
+
+    Parameters
+    ----------
+    estimator : object type that implements the "fit" and "predict" methods
+        An object of that type which is cloned for each validation.
+
+    title : string
+        Title for the chart.
+
+    X : array-like, shape (n_samples, n_features)
+        Training vector, where n_samples is the number of samples and
+        n_features is the number of features.
+
+    y : array-like, shape (n_samples) or (n_samples, n_features), optional
+        Target relative to X for classification or regression;
+        None for unsupervised learning.
+
+    ylim : tuple, shape (ymin, ymax), optional
+        Defines minimum and maximum yvalues plotted.
+
+    cv : int, cross-validation generator or an iterable, optional
+        Determines the cross-validation splitting strategy.
+        Possible inputs for cv are:
+          - None, to use the default 3-fold cross-validation,
+          - integer, to specify the number of folds.
+          - :term:`CV splitter`,
+          - An iterable yielding (train, test) splits as arrays of indices.
+
+        For integer/None inputs, if ``y`` is binary or multiclass,
+        :class:`StratifiedKFold` used. If the estimator is not a classifier
+        or if ``y`` is neither binary nor multiclass, :class:`KFold` is used.
+
+        Refer :ref:`User Guide <cross_validation>` for the various
+        cross-validators that can be used here.
+
+    n_jobs : int or None, optional (default=None)
+        Number of jobs to run in parallel.
+        ``None`` means 1 unless in a :obj:`joblib.parallel_backend` context.
+        ``-1`` means using all processors. See :term:`Glossary <n_jobs>`
+        for more details.
+
+    train_sizes : array-like, shape (n_ticks,), dtype float or int
+        Relative or absolute numbers of training examples that will be used to
+        generate the learning curve. If the dtype is float, it is regarded as a
+        fraction of the maximum size of the training set (that is determined
+        by the selected validation method), i.e. it has to be within (0, 1].
+        Otherwise it is interpreted as absolute sizes of the training sets.
+        Note that for classification the number of samples usually have to
+        be big enough to contain at least one sample from each class.
+        (default: np.linspace(0.1, 1.0, 5))
+    """
+    
+    
+    train_sizes, train_scores, test_scores = learning_curve(estimator
+                                                            ,X,y
+                                                            ,cv=cv
+                                                            ,scoring=scoring
+                                                            ,n_jobs=n_jobs
+                                                            ,train_sizes=train_sizes
+                                                            ,verbose=verbose
+                                                           )
+    train_scores_mean = np.mean(train_scores, axis=1)
+    train_scores_std = np.std(train_scores, axis=1)
+    test_scores_mean = np.mean(test_scores, axis=1)
+    test_scores_std = np.std(test_scores, axis=1)
+    
+
+    ax.fill_between(train_sizes, train_scores_mean - train_scores_std,
+                     train_scores_mean + train_scores_std, alpha=0.1,
+                     color="r")
+    ax.fill_between(train_sizes, test_scores_mean - test_scores_std,
+                     test_scores_mean + test_scores_std, alpha=0.1, color="g")
+    ax.plot(train_sizes, train_scores_mean, 'o-', color="r",
+             label="Training score")
+    ax.plot(train_sizes, test_scores_mean, 'o-', color="g",
+             label="Cross-validation score")
+
+    ## Annotate Plots
+    ax.title.set_text("Learning Curve: " + title_suffix)
+    if (ylim == None):
+        ax.set_ylim(-0.1, 1.1)
+    if ylim is not None:
+        ax.set_ylim(*ylim)
+       
+    ax.set_ylabel(title_suffix + " Score")
+    ax.set_xlabel("Training examples")
+    ax.grid(b=True)
+    ax.legend(loc="best")
+    #return plt
+
+
+def plot_learning_curves(estimator,X,y,scoring,cv=None,n_jobs=None,verbose=1,arFigsize=None):
+    num_rows = math.ceil(len(scoring)/2)
+    if (arFigsize == None):
+        arFigsize = (12,num_rows*4)
+    
+    fig, axes = plt.subplots(num_rows, 2, figsize=arFigsize, squeeze=False)
+    i = 0
+    for scorer in scoring.keys():
+        with time_usage(" Learning Curve | " + scorer):
+            plot_learning_curve(estimator=estimator
+                                ,title_suffix  = scorer.title()
+                                ,X=X, y=y
+                                ,scoring=scoring[scorer]
+                                ,cv=cv
+                                ,n_jobs=n_jobs
+                                ,verbose=verbose
+                                ,ax=axes[math.floor(i/2),i%2]
+                               )
+            i = i+1
+
+   #plt.show()