Merge pull request #4571 from PrimozGodec/test-and-score-doc

Test and Score: Update documentation

Author: ajdapretnar

Orange/widgets/evaluate/owtestlearners.py Orange/widgets/evaluate/owtestandscore.pyOrange/widgets/evaluate/owtestlearners.py renamed to Orange/widgets/evaluate/owtestandscore.py

@@ -114,7 +114,7 @@ def raise_(exc):
 
 class State(enum.Enum):
     """
-    OWTestLearner's runtime state.
+    OWTestAndScore's runtime state.
     """
     #: No or insufficient input (i.e. no data or no learners)
     Waiting = "Waiting"
@@ -126,12 +126,13 @@ class State(enum.Enum):
     Cancelled = "Cancelled"
 
 
-class OWTestLearners(OWWidget):
+class OWTestAndScore(OWWidget):
     name = "Test and Score"
     description = "Cross-validation accuracy estimation."
     icon = "icons/TestLearners1.svg"
     priority = 100
     keywords = ['Cross Validation', 'CV']
+    replaces = ["Orange.widgets.evaluate.owtestlearners.OWTestLearners"]
 
     class Inputs:
         train_data = Input("Data", Table, default=True)
@@ -345,10 +346,10 @@ def _update_controls(self):
                 self.fold_feature = self.feature_model[0]
         enabled = bool(self.feature_model)
         self.controls.resampling.buttons[
-            OWTestLearners.FeatureFold].setEnabled(enabled)
+            OWTestAndScore.FeatureFold].setEnabled(enabled)
         self.features_combo.setEnabled(enabled)
-        if self.resampling == OWTestLearners.FeatureFold and not enabled:
-            self.resampling = OWTestLearners.KFold
+        if self.resampling == OWTestAndScore.FeatureFold and not enabled:
+            self.resampling = OWTestAndScore.KFold
 
     @Inputs.learner
     def set_learner(self, learner, key):
@@ -428,7 +429,7 @@ def set_train_data(self, data):
             self._update_class_selection()
             self.openContext(data.domain)
             if self.fold_feature_selected and bool(self.feature_model):
-                self.resampling = OWTestLearners.FeatureFold
+                self.resampling = OWTestAndScore.FeatureFold
         self._invalidate()
 
     @Inputs.test_data
@@ -471,7 +472,7 @@ def set_test_data(self, data):
             self.Warning.missing_data.clear()
 
         self.test_data = data
-        if self.resampling == OWTestLearners.TestOnTest:
+        if self.resampling == OWTestAndScore.TestOnTest:
             self._invalidate()
 
     def _which_missing_data(self):
@@ -533,26 +534,26 @@ def handleNewSignals(self):
             self.__update()
 
     def kfold_changed(self):
-        self.resampling = OWTestLearners.KFold
+        self.resampling = OWTestAndScore.KFold
         self._param_changed()
 
     def fold_feature_changed(self):
-        self.resampling = OWTestLearners.FeatureFold
+        self.resampling = OWTestAndScore.FeatureFold
         self._param_changed()
 
     def shuffle_split_changed(self):
-        self.resampling = OWTestLearners.ShuffleSplit
+        self.resampling = OWTestAndScore.ShuffleSplit
         self._param_changed()
 
     def _param_changed(self):
-        self.modcompbox.setEnabled(self.resampling == OWTestLearners.KFold)
+        self.modcompbox.setEnabled(self.resampling == OWTestAndScore.KFold)
         self._update_view_enabled()
         self._invalidate()
         self.__update()
 
     def _update_view_enabled(self):
         self.comparison_table.setEnabled(
-            self.resampling == OWTestLearners.KFold
+            self.resampling == OWTestAndScore.KFold
             and len(self.learners) > 1
             and self.data is not None)
         self.score_table.view.setEnabled(
@@ -661,7 +662,7 @@ def update_comparison_table(self):
             return
         names = [learner_name(slot.learner) for slot in slots]
         self._set_comparison_headers(names)
-        if self.resampling == OWTestLearners.KFold:
+        if self.resampling == OWTestAndScore.KFold:
             scores = self._scores_by_folds(slots)
             self._fill_table(names, scores)
 
@@ -790,7 +791,7 @@ def _on_target_class_changed(self):
     def _invalidate(self, which=None):
         self.cancel()
         self.fold_feature_selected = \
-            self.resampling == OWTestLearners.FeatureFold
+            self.resampling == OWTestAndScore.FeatureFold
         # Invalidate learner results for `which` input keys
         # (if None then all learner results are invalidated)
         if which is None:
@@ -912,14 +913,14 @@ def __update(self):
             self.__state = State.Waiting
             self.commit()
             return
-        if self.resampling == OWTestLearners.KFold and \
+        if self.resampling == OWTestAndScore.KFold and \
                 len(self.data) < self.NFolds[self.n_folds]:
             self.Error.too_many_folds()
             self.__state = State.Waiting
             self.commit()
             return
 
-        elif self.resampling == OWTestLearners.TestOnTest:
+        elif self.resampling == OWTestAndScore.TestOnTest:
             if self.test_data is None:
                 if not self.Error.test_data_empty.is_shown():
                     self.Warning.test_data_missing()
@@ -947,30 +948,30 @@ def __update(self):
         # learners bellow)
         learners_c = [copy.deepcopy(learner) for learner in learners]
 
-        if self.resampling == OWTestLearners.TestOnTest:
+        if self.resampling == OWTestAndScore.TestOnTest:
             test_f = partial(
                 Orange.evaluation.TestOnTestData(
                     store_data=True, store_models=True),
                 self.data, self.test_data, learners_c, self.preprocessor
             )
         else:
-            if self.resampling == OWTestLearners.KFold:
+            if self.resampling == OWTestAndScore.KFold:
                 sampler = Orange.evaluation.CrossValidation(
                     k=self.NFolds[self.n_folds],
                     random_state=rstate)
-            elif self.resampling == OWTestLearners.FeatureFold:
+            elif self.resampling == OWTestAndScore.FeatureFold:
                 sampler = Orange.evaluation.CrossValidationFeature(
                     feature=self.fold_feature)
-            elif self.resampling == OWTestLearners.LeaveOneOut:
+            elif self.resampling == OWTestAndScore.LeaveOneOut:
                 sampler = Orange.evaluation.LeaveOneOut()
-            elif self.resampling == OWTestLearners.ShuffleSplit:
+            elif self.resampling == OWTestAndScore.ShuffleSplit:
                 sampler = Orange.evaluation.ShuffleSplit(
                     n_resamples=self.NRepeats[self.n_repeats],
                     train_size=self.SampleSizes[self.sample_size] / 100,
                     test_size=None,
                     stratified=self.shuffle_stratified,
                     random_state=rstate)
-            elif self.resampling == OWTestLearners.TestOnTrain:
+            elif self.resampling == OWTestAndScore.TestOnTrain:
                 sampler = Orange.evaluation.TestOnTrainingData(
                     store_models=True)
             else:
@@ -1184,7 +1185,7 @@ def results_one_vs_rest(results, pos_index):
                          Orange.regression.KNNRegressionLearner(),
                          Orange.regression.RidgeRegressionLearner()]
 
-    WidgetPreview(OWTestLearners).run(
+    WidgetPreview(OWTestAndScore).run(
         set_train_data=preview_data,
         set_test_data=preview_data,
         set_learner=[(learner, i) for i, learner in enumerate(prev_learners)]

…ts/evaluate/tests/test_owtestlearners.py …ts/evaluate/tests/test_owtestandscore.pyOrange/widgets/evaluate/tests/test_owtestlearners.py renamed to Orange/widgets/evaluate/tests/test_owtestandscore.py Orange/widgets/evaluate/tests/test_owtestlearners.py renamed to Orange/widgets/evaluate/tests/test_owtestandscore.py

@@ -18,8 +18,8 @@
 from Orange.base import Learner
 from Orange.modelling import ConstantLearner
 from Orange.regression import MeanLearner
-from Orange.widgets.evaluate.owtestlearners import (
-    OWTestLearners, results_one_vs_rest)
+from Orange.widgets.evaluate.owtestandscore import (
+    OWTestAndScore, results_one_vs_rest)
 from Orange.widgets.evaluate.utils import BUILTIN_SCORERS_ORDER
 from Orange.widgets.settings import (
     ClassValuesContextHandler, PerfectDomainContextHandler)
@@ -33,10 +33,10 @@ def fit(self, *_, **_2):  # pylint: disable=arguments-differ
         return 1 / 0
 
 
-class TestOWTestLearners(WidgetTest):
+class TestOWTestAndScore(WidgetTest):
     def setUp(self):
         super().setUp()
-        self.widget = self.create_widget(OWTestLearners)  # type: OWTestLearners
+        self.widget = self.create_widget(OWTestAndScore)  # type: OWTestAndScore
 
         self.scores_domain = Domain(
             [ContinuousVariable("a"), ContinuousVariable("b")],
@@ -82,7 +82,7 @@ def test_more_learners(self):
     def test_testOnTest(self):
         data = Table("iris")
         self.send_signal(self.widget.Inputs.train_data, data)
-        self.widget.resampling = OWTestLearners.TestOnTest
+        self.widget.resampling = OWTestAndScore.TestOnTest
         self.send_signal(self.widget.Inputs.test_data, data)
 
     def test_testOnTest_incompatible_domain(self):
@@ -91,7 +91,7 @@ def test_testOnTest_incompatible_domain(self):
         self.send_signal(self.widget.Inputs.learner, LogisticRegressionLearner(), 0)
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertFalse(self.widget.Error.test_data_incompatible.is_shown())
-        self.widget.resampling = OWTestLearners.TestOnTest
+        self.widget.resampling = OWTestAndScore.TestOnTest
         # test data with the same class (otherwise the widget shows a different error)
         # and a non-nan X
         iris_test = iris.transform(Domain([ContinuousVariable("x")],
@@ -106,7 +106,7 @@ def test_CrossValidationByFeature(self):
         attrs = data.domain.attributes
         domain = Domain(attrs[:-1], attrs[-1], data.domain.class_vars)
         data_with_disc_metas = Table.from_table(domain, data)
-        rb = self.widget.controls.resampling.buttons[OWTestLearners.FeatureFold]
+        rb = self.widget.controls.resampling.buttons[OWTestAndScore.FeatureFold]
 
         self.send_signal(self.widget.Inputs.learner, ConstantLearner(), 0)
         self.send_signal(self.widget.Inputs.train_data, data)
@@ -117,15 +117,15 @@ def test_CrossValidationByFeature(self):
         self.send_signal(self.widget.Inputs.train_data, data_with_disc_metas)
         self.assertTrue(rb.isEnabled())
         rb.click()
-        self.assertEqual(self.widget.resampling, OWTestLearners.FeatureFold)
+        self.assertEqual(self.widget.resampling, OWTestAndScore.FeatureFold)
         self.assertTrue(self.widget.features_combo.isEnabled())
         self.assertEqual(self.widget.features_combo.currentText(), "iris")
         self.assertEqual(len(self.widget.features_combo.model()), 1)
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
 
         self.send_signal(self.widget.Inputs.train_data, None)
         self.assertFalse(rb.isEnabled())
-        self.assertEqual(self.widget.resampling, OWTestLearners.KFold)
+        self.assertEqual(self.widget.resampling, OWTestAndScore.KFold)
         self.assertFalse(self.widget.features_combo.isEnabled())
 
     def test_migrate_removes_invalid_contexts(self):
@@ -226,7 +226,7 @@ class NewRegressionScore(RegressionScore):
 
     def test_target_changing(self):
         data = Table("iris")
-        w = self.widget  #: OWTestLearners
+        w = self.widget  #: OWTestAndScore
         model = w.score_table.model
 
         w.n_folds = 2
@@ -305,7 +305,7 @@ def _retrieve_scores(self):
         return auc, ca, f1, precision, recall
 
     def _test_scores(self, train_data, test_data, learner, sampling, n_folds):
-        w = self.widget  #: OWTestLearners
+        w = self.widget  #: OWTestAndScore
         w.controls.resampling.buttons[sampling].click()
         if n_folds is not None:
             w.n_folds = n_folds
@@ -323,7 +323,7 @@ def test_scores_constant_all_same(self):
         )
 
         self.assertTupleEqual(self._test_scores(
-            table, table, ConstantLearner(), OWTestLearners.TestOnTest, None),
+            table, table, ConstantLearner(), OWTestAndScore.TestOnTest, None),
                               (None, 1, 1, 1, 1))
 
     def test_scores_log_reg_overfitted(self):
@@ -334,7 +334,7 @@ def test_scores_log_reg_overfitted(self):
 
         self.assertTupleEqual(self._test_scores(
             table, table, LogisticRegressionLearner(),
-            OWTestLearners.TestOnTest, None),
+            OWTestAndScore.TestOnTest, None),
                               (1, 1, 1, 1, 1))
 
     def test_scores_log_reg_bad(self):
@@ -349,7 +349,7 @@ def test_scores_log_reg_bad(self):
 
         self.assertTupleEqual(self._test_scores(
             table_train, table_test, LogisticRegressionLearner(),
-            OWTestLearners.TestOnTest, None),
+            OWTestAndScore.TestOnTest, None),
                               (0, 0, 0, 0, 0))
 
     def test_scores_log_reg_bad2(self):
@@ -361,7 +361,7 @@ def test_scores_log_reg_bad2(self):
             list(zip(*(self.scores_table_values + [list("yynn")]))))
         self.assertTupleEqual(self._test_scores(
             table_train, table_test, LogisticRegressionLearner(),
-            OWTestLearners.TestOnTest, None),
+            OWTestAndScore.TestOnTest, None),
                               (0, 0, 0, 0, 0))
 
     def test_scores_log_reg_advanced(self):
@@ -377,7 +377,7 @@ def test_scores_log_reg_advanced(self):
         np.testing.assert_almost_equal(
             self._test_scores(table_train, table_test,
                               LogisticRegressionLearner(),
-                              OWTestLearners.TestOnTest, None),
+                              OWTestAndScore.TestOnTest, None),
             (2 / 3, 0.8, 0.8, 13 / 15, 0.8))
 
     def test_scores_cross_validation(self):
@@ -388,7 +388,7 @@ def test_scores_cross_validation(self):
             all(x >= y for x, y in zip(
                 self._test_scores(
                     Table("iris")[::15], None, LogisticRegressionLearner(),
-                    OWTestLearners.KFold, 0),
+                    OWTestAndScore.KFold, 0),
                 (0.8, 0.5, 0.5, 0.5, 0.5))))
 
     def test_no_pregressbar_warning(self):
@@ -426,23 +426,23 @@ def test_comparison_requires_cv(self):
         self._set_three_majorities()
         baycomp.two_on_single.reset_mock()
 
-        rbs[OWTestLearners.KFold].click()
+        rbs[OWTestAndScore.KFold].click()
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))
         self.assertTrue(w.modcompbox.isEnabled())
         self.assertTrue(w.comparison_table.isEnabled())
         baycomp.two_on_single.assert_called()
         baycomp.two_on_single.reset_mock()
 
-        rbs[OWTestLearners.LeaveOneOut].click()
+        rbs[OWTestAndScore.LeaveOneOut].click()
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertIsNone(w.comparison_table.cellWidget(0, 1))
         self.assertFalse(w.modcompbox.isEnabled())
         self.assertFalse(w.comparison_table.isEnabled())
         baycomp.two_on_single.assert_not_called()
         baycomp.two_on_single.reset_mock()
 
-        rbs[OWTestLearners.KFold].click()
+        rbs[OWTestAndScore.KFold].click()
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertIsNotNone(w.comparison_table.cellWidget(0, 1))
         self.assertTrue(w.modcompbox.isEnabled())
@@ -457,7 +457,7 @@ def test_comparison_requires_multiple_models(self):
 
         self._set_three_majorities()
 
-        rbs[OWTestLearners.KFold].click()
+        rbs[OWTestAndScore.KFold].click()
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertTrue(w.comparison_table.isEnabled())
 
@@ -469,7 +469,7 @@ def test_comparison_requires_multiple_models(self):
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertFalse(w.comparison_table.isEnabled())
 
-        rbs[OWTestLearners.LeaveOneOut].click()
+        rbs[OWTestAndScore.LeaveOneOut].click()
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertFalse(w.comparison_table.isEnabled())
 
@@ -479,7 +479,7 @@ def test_comparison_requires_multiple_models(self):
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertFalse(w.comparison_table.isEnabled())
 
-        rbs[OWTestLearners.KFold].click()
+        rbs[OWTestAndScore.KFold].click()
         self.get_output(self.widget.Outputs.evaluations_results, wait=5000)
         self.assertTrue(w.comparison_table.isEnabled())
 
@@ -549,7 +549,7 @@ def test_comparison_binary_score(self):
             self.assertFalse("average" in kwargs)
 
             simulate.combobox_activate_item(w.controls.class_selection,
-                                            OWTestLearners.TARGET_AVERAGE)
+                                            OWTestAndScore.TARGET_AVERAGE)
             _, kwargs = f1mock.call_args
             self.assertEqual(kwargs["average"], "weighted")
             self.assertFalse("target" in kwargs)

Orange/widgets/report/tests/test_report.py

@@ -17,7 +17,7 @@
 from Orange.widgets.evaluate.owcalibrationplot import OWCalibrationPlot
 from Orange.widgets.evaluate.owliftcurve import OWLiftCurve
 from Orange.widgets.evaluate.owrocanalysis import OWROCAnalysis
-from Orange.widgets.evaluate.owtestlearners import OWTestLearners
+from Orange.widgets.evaluate.owtestandscore import OWTestAndScore
 from Orange.widgets.unsupervised.owcorrespondence import OWCorrespondenceAnalysis
 from Orange.widgets.unsupervised.owdistancemap import OWDistanceMap
 from Orange.widgets.unsupervised.owdistances import OWDistances
@@ -66,7 +66,7 @@ class TestReportWidgets(WidgetTest):
                     OWMDS, OWPCA]
     dist_widgets = [OWDistanceMap, OWHierarchicalClustering]
     visu_widgets = VISUALIZATION_WIDGETS
-    spec_widgets = [OWTestLearners, OWTreeGraph]
+    spec_widgets = [OWTestAndScore, OWTreeGraph]
 
     def _create_report(self, widgets, rep, data):
         for widget in widgets:
@@ -108,7 +108,7 @@ def test_report_widgets_evaluate(self):
                                   k=3)
         results.learner_names = ["LR l2"]
 
-        w = self.create_widget(OWTestLearners)
+        w = self.create_widget(OWTestAndScore)
         set_learner = getattr(w, w.Inputs.learner.handler)
         set_train = getattr(w, w.Inputs.train_data.handler)
         set_test = getattr(w, w.Inputs.test_data.handler)