OWTestLearners: Cross validation by feature

Author: VesnaT

Orange/widgets/evaluate/owtestlearners.py

@@ -11,7 +11,7 @@
 from AnyQt.QtGui import QStandardItemModel, QStandardItem
 from AnyQt.QtCore import Qt, QSize
 
-from Orange.data import Table
+from Orange.data import Table, DiscreteVariable
 from Orange.data.sql.table import SqlTable, AUTO_DL_LIMIT
 import Orange.evaluation
 import Orange.classification
@@ -22,6 +22,7 @@
 from Orange.preprocess.preprocess import Preprocess
 from Orange.preprocess import RemoveNaNClasses
 from Orange.widgets import widget, gui, settings
+from Orange.widgets.utils.itemmodels import DomainModel
 from Orange.widgets.widget import OWWidget, Msg
 
 Input = namedtuple(
@@ -137,12 +138,12 @@ class OWTestLearners(OWWidget):
     outputs = [("Predictions", Table),
                ("Evaluation Results", Results)]
 
-    settingsHandler = settings.ClassValuesContextHandler()
+    settingsHandler = settings.PerfectDomainContextHandler(metas_in_res=True)
 
     #: Resampling/testing types
     KFold, ShuffleSplit, LeaveOneOut, TestOnTrain, TestOnTest = 0, 1, 2, 3, 4
     #: Numbers of folds
-    NFolds = [2, 3, 5, 10, 20]
+    NFolds = [2, 3, 5, 10, 20, "From feature"]
     #: Number of repetitions
     NRepeats = [2, 3, 5, 10, 20, 50, 100]
     #: Sample sizes
@@ -160,6 +161,8 @@ class OWTestLearners(OWWidget):
     sample_size = settings.Setting(9)
     #: Stratified sampling for Random Sampling
     shuffle_stratified = settings.Setting(True)
+    # CV where nr. of feature values determines nr. of folds
+    fold_feature = settings.ContextSetting(None)
 
     TARGET_AVERAGE = "(Average over classes)"
     class_selection = settings.ContextSetting(TARGET_AVERAGE)
@@ -204,13 +207,18 @@ def __init__(self):
 
         gui.appendRadioButton(rbox, "Cross validation")
         ibox = gui.indentedBox(rbox)
-        gui.comboBox(
+        self.n_folds_combo = gui.comboBox(
             ibox, self, "n_folds", label="Number of folds: ",
             items=[str(x) for x in self.NFolds], maximumContentsLength=3,
             orientation=Qt.Horizontal, callback=self.kfold_changed)
-        gui.checkBox(
+        self.stratified_check = gui.checkBox(
             ibox, self, "cv_stratified", "Stratified",
             callback=self.kfold_changed)
+        self.feature_model = DomainModel(
+            order=DomainModel.METAS, valid_types=DiscreteVariable)
+        self.features_combo = gui.comboBox(
+            ibox, self, "fold_feature", model=self.feature_model,
+            orientation=Qt.Horizontal, callback=self.fold_feature_changed)
 
         gui.appendRadioButton(rbox, "Random sampling")
         ibox = gui.indentedBox(rbox)
@@ -257,9 +265,32 @@ def __init__(self):
         box = gui.vBox(self.mainArea, "Evaluation Results")
         box.layout().addWidget(self.view)
 
+    @property
+    def kfold_feature_index(self):
+        return len(self.NFolds) - 1
+
     def sizeHint(self):
         return QSize(780, 1)
 
+    def __hide_show_feature_combo(self):
+        cv_feature = self.n_folds == self.kfold_feature_index
+        self.stratified_check.setVisible(not cv_feature)
+        self.features_combo.setVisible(cv_feature)
+        if self.fold_feature is None and cv_feature and self.feature_model:
+            self.fold_feature = self.feature_model[0]
+
+    def _update_controls(self):
+        self.fold_feature = None
+        self.feature_model.set_domain(None)
+        if self.data:
+            self.feature_model.set_domain(self.data.domain)
+        enable = bool(self.feature_model)
+        item = self.n_folds_combo.model().item(self.kfold_feature_index)
+        item.setEnabled(enable)
+        if self.n_folds == self.kfold_feature_index and not enable:
+            self.n_folds = 3
+        self.__hide_show_feature_combo()
+
     def set_learner(self, learner, key):
         """
         Set the input `learner` for `key`.
@@ -310,9 +341,10 @@ def set_train_data(self, data):
 
         self.data = data
         self.closeContext()
+        self._update_controls()
         if data is not None:
             self._update_class_selection()
-            self.openContext(data.domain.class_var)
+            self.openContext(data.domain)
         self._invalidate()
 
     def set_test_data(self, data):
@@ -372,6 +404,10 @@ def handleNewSignals(self):
 
     def kfold_changed(self):
         self.resampling = OWTestLearners.KFold
+        self.__hide_show_feature_combo()
+        self._param_changed()
+
+    def fold_feature_changed(self):
         self._param_changed()
 
     def shuffle_split_changed(self):
@@ -429,17 +465,22 @@ def update_progress(finished):
 
         with self.progressBar():
             try:
-                folds = self.NFolds[self.n_folds]
                 if self.resampling == OWTestLearners.KFold:
-                    if len(self.data) < folds:
-                        self.Error.too_many_folds()
-                        return
-                    warnings = []
-                    results = Orange.evaluation.CrossValidation(
-                        self.data, learners, k=folds,
-                        random_state=rstate, warnings=warnings, **common_args)
-                    if warnings:
-                        self.warning(warnings[0])
+                    if self.n_folds == self.kfold_feature_index:
+                        results = Orange.evaluation.CrossValidationFeature(
+                            self.data, learners, self.fold_feature,
+                            **common_args)
+                    else:
+                        folds = self.NFolds[self.n_folds]
+                        if len(self.data) < folds:
+                            self.Error.too_many_folds()
+                            return
+                        warnings = []
+                        results = Orange.evaluation.CrossValidation(
+                            self.data, learners, k=folds,
+                            random_state=rstate, warnings=warnings, **common_args)
+                        if warnings:
+                            self.warning(warnings[0])
                 elif self.resampling == OWTestLearners.LeaveOneOut:
                     results = Orange.evaluation.LeaveOneOut(
                         self.data, learners, **common_args)

Orange/widgets/evaluate/tests/test_owtestlearners.py

@@ -3,9 +3,10 @@
 
 import unittest
 
-from Orange.data import Table
+from Orange.data import Table, Domain
 from Orange.classification import MajorityLearner
 from Orange.regression import MeanLearner
+from Orange.modelling import ConstantLearner
 
 from Orange.evaluation import Results, TestOnTestData
 from Orange.widgets.tests.base import WidgetTest
@@ -38,6 +39,31 @@ def test_basic(self):
         self.assertIsNotNone(res.domain)
         self.assertIsNotNone(res.data)
 
+    def test_feature_cv_combo(self):
+        data, i = Table("iris"), 5
+        attrs = data.domain.attributes
+        domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)
+        data_with_disc_metas = Table.from_table(domain, data)
+
+        self.send_signal("Learner", ConstantLearner(), 0)
+        self.send_signal("Data", data)
+        self.assertFalse(self.widget.n_folds_combo.model().item(i).isEnabled())
+        self.assertTrue(self.widget.features_combo.isHidden())
+        self.assertFalse(self.widget.stratified_check.isHidden())
+
+        self.send_signal("Data", data_with_disc_metas)
+        self.assertTrue(self.widget.n_folds_combo.model().item(i).isEnabled())
+        self.widget.n_folds_combo.activated.emit(i)
+        self.widget.n_folds_combo.setCurrentIndex(i)
+        self.assertTrue(self.widget.stratified_check.isHidden())
+        self.assertFalse(self.widget.features_combo.isHidden())
+        self.assertEqual(len(self.widget.features_combo.model()), 1)
+
+        self.send_signal("Data", None)
+        self.assertFalse(self.widget.n_folds_combo.model().item(i).isEnabled())
+        self.assertTrue(self.widget.features_combo.isHidden())
+        self.assertFalse(self.widget.stratified_check.isHidden())
+
 
 class TestHelpers(unittest.TestCase):
     def test_results_one_vs_rest(self):
@@ -64,4 +90,4 @@ def test_results_one_vs_rest(self):
 
         np.testing.assert_equal(r1.row_indices, res.row_indices)
         np.testing.assert_equal(r2.row_indices, res.row_indices)
-        np.testing.assert_equal(r3.row_indices, res.row_indices)
+        np.testing.assert_equal(r3.row_indices, res.row_indices)