update binary outcome handling

Author: bdwilliamson

vimpy/cv_vim.py

@@ -18,12 +18,13 @@ class cv_vim:
     @param measure_type the predictiveness measure to use (for now, one of "r_squared", "auc", "accuracy", "deviance")
     @param V the number of cross-fitting folds (defaults to 5)
     @param pred_func the function that predicts outcome given features
+    @param ensemble is pred_func an ensemble (True) or a single function (False, default)
     @param f fitted values from regression of outcome on all features (only used if pred_func is not specified)
     @param r fitted values from regression of outcome on reduced set of features (only used if pred_func is not specified)
     @param folds a list of length 3: outer folds, for hypothesis testing; inner folds based on the outer folds == 1 (for cross-fitting); inner folds based on outer folds == 0 (for cross-fitting)
     @param na_rm remove NAs prior to computing predictiveness? (defaults to False)
     """
-    def __init__(self, y, x, s, measure_type, V = 5, pred_func = None, f = None, r = None, folds = None, na_rm = False):
+    def __init__(self, y, x, s, measure_type, V = 5, pred_func = None, ensemble = False, f = None, r = None, folds = None, na_rm = False):
         self.y_ = y
         self.x_ = x
         self.s_ = s
@@ -67,6 +68,7 @@ def __init__(self, y, x, s, measure_type, V = 5, pred_func = None, f = None, r =
         ## if only two unique values in y, assume binary
         self.binary_ = (np.unique(y).shape[0] == 2)
         self.na_rm_ = na_rm
+        self.ensemble_ = ensemble
 
     ## calculate the plug-in estimator
     def get_point_est(self):
@@ -82,8 +84,8 @@ def get_point_est(self):
             this_redu_func = self.r_
             folds_1 = self.folds_inner_1
             folds_0 = self.folds_inner_0
-        self.v_full_, self.preds_full_, ic_full, self.folds_inner_1, self.cc_1 = predictiveness_func(self.x_[self.folds_outer_ == 1, :], self.y_[self.folds_outer_ == 1], np.arange(self.p_), self.measure_, this_full_func, V = self.V_, stratified = self.binary_, na_rm = self.na_rm_, folds = folds_1)
-        self.v_redu_, self.preds_redu_, ic_redu, self.folds_inner_0, self.cc_0 = predictiveness_func(self.x_[self.folds_outer_ == 0, :], self.y_[self.folds_outer_ == 0], np.delete(np.arange(self.p_), self.s_), self.measure_, this_redu_func, V = self.V_, stratified = self.binary_, na_rm = self.na_rm_, folds = folds_0)
+        self.v_full_, self.preds_full_, ic_full, self.folds_inner_1, self.cc_1 = predictiveness_func(self.x_[self.folds_outer_ == 1, :], self.y_[self.folds_outer_ == 1], np.arange(self.p_), self.measure_, this_full_func, V = self.V_, stratified = self.binary_, na_rm = self.na_rm_, folds = folds_1, ensemble = self.ensemble_)
+        self.v_redu_, self.preds_redu_, ic_redu, self.folds_inner_0, self.cc_0 = predictiveness_func(self.x_[self.folds_outer_ == 0, :], self.y_[self.folds_outer_ == 0], np.delete(np.arange(self.p_), self.s_), self.measure_, this_redu_func, V = self.V_, stratified = self.binary_, na_rm = self.na_rm_, folds = folds_0, ensemble = self.ensemble_)
         self.vimp_ = self.v_full_ - self.v_redu_
         self.ic_full_[:ic_full.shape[0]] = ic_full
         self.ic_redu_[:ic_redu.shape[0]] = ic_redu

vimpy/predictiveness_measures.py

@@ -49,7 +49,7 @@ def cv_predictiveness(x, y, S, measure, pred_func, V = 5, stratified = True, na_
             if measure.__name__ in ["r_squared"]:
                 preds_v = pred_func.predict(x_train[:, S])
             else:
-                preds_v = pred_func.predict_proba(x_train[:, S])
+                preds_v = pred_func.predict_proba(x_train[:, S])[:, 1]
         preds[cc_cond] = preds_v
         vs[0] = measure(y_train, preds_v)
         ics[cc_cond] = compute_ic(y_train, preds_v, measure.__name__)
@@ -73,7 +73,7 @@ def cv_predictiveness(x, y, S, measure, pred_func, V = 5, stratified = True, na_
 
 
 # general predictiveness based on precomputed fits
-def cv_predictiveness_precomputed(x, y, S, measure, f, V = 5, stratified = True, folds = None, na_rm = False):
+def cv_predictiveness_precomputed(x, y, S, measure, f, V = 5, stratified = True, folds = None, na_rm = False, ensemble = False):
     """
     Compute a cross-validated measure of predictiveness based on the data, the chosen measure, and the sets of fitted values f and r
 
@@ -86,6 +86,7 @@ def cv_predictiveness_precomputed(x, y, S, measure, f, V = 5, stratified = True,
     @param stratified: should the folds be stratified?
     @param folds: the CV folds
     @param na_rm: should we do a complete-case analysis (True) or not (False)
+    @param ensemble: is this an ensemble or not (dummy)
 
     @return cross-validated measure of predictiveness, along with preds and ics
     """

vimpy/vim.py

@@ -17,14 +17,15 @@ class vim:
     @param s the feature group of interest
     @param measure_type the predictiveness measure to use (for now, one of "r_squared", "auc", "accuracy", "deviance")
     @param pred_func the function that predicts outcome given features
+    @param ensemble is pred_func an ensemble (True) or a single function (False, default)
     @param f fitted values from regression of outcome on all features (only used if pred_func is not specified)
     @param r fitted values from regression of outcome on reduced set of features (only used if pred_func is not specified)
     @param folds outer folds, for hypothesis testing (only used if pred_func is not specified)
     @param na_rm remove NAs prior to computing predictiveness? (defaults to False)
 
     @return an object of class vim
     """
-    def __init__(self, y, x, s, measure_type, pred_func = None, f = None, r = None, folds = None, na_rm = False):
+    def __init__(self, y, x, s, measure_type, pred_func = None, ensemble = False, f = None, r = None, folds = None, na_rm = False):
         self.y_ = y
         self.x_ = x
         self.s_ = s
@@ -66,6 +67,7 @@ def __init__(self, y, x, s, measure_type, pred_func = None, f = None, r = None,
         ## if only two unique values in y, assume binary
         self.binary_ = (np.unique(y).shape[0] == 2)
         self.na_rm_ = na_rm
+        self.ensemble_ = ensemble
 
     ## calculate the variable importance estimate
     def get_point_est(self):
@@ -79,8 +81,8 @@ def get_point_est(self):
             this_full_func = self.f_
             this_redu_func = self.r_
             folds = None
-        self.v_full_, self.preds_full_, ic_full, self.folds_inner_1, self.cc_1 = predictiveness_func(self.x_[self.folds_outer_ == 1, :], self.y_[self.folds_outer_ == 1], np.arange(self.p_), self.measure_, this_full_func, V = 1, stratified = self.binary_, na_rm = self.na_rm_, folds = folds)
-        self.v_redu_, self.preds_redu_, ic_redu, self.folds_inner_0, self.cc_0 = predictiveness_func(self.x_[self.folds_outer_ == 0, :], self.y_[self.folds_outer_ == 0], np.delete(np.arange(self.p_), self.s_), self.measure_, this_redu_func, V = 1, stratified = self.binary_, na_rm = self.na_rm_, folds = folds)
+        self.v_full_, self.preds_full_, ic_full, self.folds_inner_1, self.cc_1 = predictiveness_func(self.x_[self.folds_outer_ == 1, :], self.y_[self.folds_outer_ == 1], np.arange(self.p_), self.measure_, this_full_func, V = 1, stratified = self.binary_, na_rm = self.na_rm_, folds = folds, ensemble = self.ensemble_)
+        self.v_redu_, self.preds_redu_, ic_redu, self.folds_inner_0, self.cc_0 = predictiveness_func(self.x_[self.folds_outer_ == 0, :], self.y_[self.folds_outer_ == 0], np.delete(np.arange(self.p_), self.s_), self.measure_, this_redu_func, V = 1, stratified = self.binary_, na_rm = self.na_rm_, folds = folds, ensemble = self.ensemble_)
         self.vimp_ = self.v_full_ - self.v_redu_
         self.ic_full_[:ic_full.shape[0]] = ic_full
         self.ic_redu_[:ic_redu.shape[0]] = ic_redu