Models: output probabilities with correct shape

Author: PrimozGodec

Orange/classification/softmax_regression.py

@@ -70,7 +70,9 @@ def cost_grad(self, Theta_flat, X, Y):
 
         return cost, grad.ravel()
 
-    def fit(self, X, y, W):
+    def fit_storage(self, data):
+        X, y = data.X, data.Y
+
         if len(y.shape) > 1:
             raise ValueError('Softmax regression does not support '
                              'multi-label classification')
@@ -81,7 +83,7 @@ def fit(self, X, y, W):
 
         X = np.hstack((X, np.ones((X.shape[0], 1))))
 
-        self.num_classes = np.unique(y).size
+        self.num_classes = len(data.domain.class_var.values)
         Y = np.eye(self.num_classes)[y.ravel().astype(int)]
 
         theta = np.zeros(self.num_classes * X.shape[1])

Orange/classification/svm.py

@@ -10,6 +10,7 @@
 
 class SVMLearner(SklLearner):
     __wraps__ = skl_svm.SVC
+    __returns__ = SklModel
     preprocessors = svm_pps
 
     def __init__(self, C=1.0, kernel='rbf', degree=3, gamma="auto",
@@ -32,19 +33,9 @@ def __init__(self, penalty='l2', loss='squared_hinge', dual=True,
         self.params = vars()
 
 
-class NuSVMClassifier(SklModel):
-
-    def predict(self, X):
-        value = self.skl_model.predict(X)
-        if self.skl_model.probability:
-            prob = self.skl_model.predict_proba(X)
-            return value, prob
-        return value
-
-
 class NuSVMLearner(SklLearner):
     __wraps__ = skl_svm.NuSVC
-    __returns__ = NuSVMClassifier
+    __returns__ = SklModel
     preprocessors = svm_pps
 
     def __init__(self, nu=0.5, kernel='rbf', degree=3, gamma="auto", coef0=0.0,
@@ -55,10 +46,11 @@ def __init__(self, nu=0.5, kernel='rbf', degree=3, gamma="auto", coef0=0.0,
 
 
 if __name__ == '__main__':
-    import Orange
+    from Orange.evaluation import CrossValidation, CA
+    from Orange.data import Table
 
-    data = Orange.data.Table('iris')
+    data_ = Table('iris')
     learners = [SVMLearner(), NuSVMLearner(), LinearSVMLearner()]
-    res = Orange.evaluation.CrossValidation(data, learners)
-    for l, ca in zip(learners, Orange.evaluation.CA(res)):
+    res = CrossValidation()(data_, learners)
+    for l, ca in zip(learners, CA()(res)):
         print("learner: {}\nCA: {}\n".format(l, ca))

Orange/tests/test_classification.py

@@ -22,8 +22,7 @@
     MajorityLearner,
     RandomForestLearner, SimpleTreeLearner, SoftmaxRegressionLearner,
     SVMLearner, LinearSVMLearner, OneClassSVMLearner, TreeLearner, KNNLearner,
-    SimpleRandomForestLearner, EllipticEnvelopeLearner,
-    SGDClassificationLearner)
+    SimpleRandomForestLearner, EllipticEnvelopeLearner)
 from Orange.classification.rules import _RuleLearner
 from Orange.data import (ContinuousVariable, DiscreteVariable,
                          Domain, Table)
@@ -33,6 +32,24 @@
 from Orange.tests import test_filename
 
 
+def all_learners():
+    classification_modules = pkgutil.walk_packages(
+        path=Orange.classification.__path__,
+        prefix="Orange.classification.",
+        onerror=lambda x: None)
+    for importer, modname, ispkg in classification_modules:
+        try:
+            module = pkgutil.importlib.import_module(modname)
+        except ImportError:
+            continue
+
+        for name, class_ in inspect.getmembers(module, inspect.isclass):
+            if (issubclass(class_, Learner) and
+                    not name.startswith('_') and
+                    'base' not in class_.__module__):
+                yield class_
+
+
 class MultiClassTest(unittest.TestCase):
     def test_unsupported(self):
         nrows = 20
@@ -194,21 +211,70 @@ def test_incompatible_domain(self):
 
     def test_result_shape(self):
         """
-        This test function will be extended for all models in on of the
-        following pull requests.
+        Test if the results shapes are correct
         """
         iris = Table('iris')
-        learner = SGDClassificationLearner()
+        for learner in all_learners():
+            with self.subTest(learner.__name__):
+                # model trained on only one value (but three in the domain)
+                try:
+                    model = learner()(iris[0:50])
+                except TypeError:
+                    # cannot be tested with default parameters
+                    continue
+
+                res = model(iris[0:50])
+                self.assertTupleEqual((50,), res.shape)
+
+                # probabilities must still be for three classes
+                res = model(iris[0:50], model.Probs)
+                self.assertTupleEqual((50, 3), res.shape)
 
-        # model trained on only one value (but three in the domain)
-        model = learner(iris)
+                # model trained on all classes and predicting with one class
+                try:
+                    model = learner()(iris[0:50])
+                except TypeError:
+                    # cannot be tested with default parameters
+                    continue
+                res = model(iris[0:50], model.Probs)
+                self.assertTupleEqual((50, 3), res.shape)
 
-        res = model(iris[0:50])
-        self.assertTupleEqual((50,), res.shape)
+    def test_result_shape_numpy(self):
+        """
+        Test whether results shapes are correct when testing on numpy data
+        """
+        iris = Table('iris')
+        for learner in all_learners():
+            with self.subTest(learner.__name__):
+                if learner.__name__ == "CN2SDLearner":
+                    # TODO: fix CN2SDLearner
+                    continue
+                try:
+                    model = learner()(iris)
+                except TypeError:
+                    # cannot be tested with default parameters
+                    continue
+                transformed_iris = model.data_to_model_domain(iris)
 
-        # probabilities must still be for three classes
-        res = model(iris[0:50], model.Probs)
-        self.assertTupleEqual((50, 3), res.shape)
+                res = model(transformed_iris.X[0:5])
+                self.assertTupleEqual((5,), res.shape)
+
+                res = model(transformed_iris.X[0:1], model.Probs)
+                self.assertTupleEqual((1, 3), res.shape)
+
+    def test_fit_one_class(self):
+        """
+        Test whether the fitting with one class only pass - before it failed
+        for some models.
+        """
+        iris = Table('iris')
+        for learner in all_learners():
+            with self.subTest(learner.__name__):
+                try:
+                    model = learner()(iris[:50])
+                except TypeError:
+                    # cannot be tested with default parameters
+                    continue
 
 
 class ExpandProbabilitiesTest(unittest.TestCase):
@@ -309,7 +375,7 @@ def test_unknown(self):
 
     def test_missing_class(self):
         table = Table(test_filename("datasets/adult_sample_missing"))
-        for learner in LearnerAccessibility().all_learners():
+        for learner in all_learners():
             try:
                 learner = learner()
                 if isinstance(learner, NuSVMLearner):
@@ -330,33 +396,15 @@ def setUp(self):
         # Convergence warnings are irrelevant for these tests
         warnings.filterwarnings("ignore", ".*", ConvergenceWarning)
 
-
-    def all_learners(self):
-        classification_modules = pkgutil.walk_packages(
-            path=Orange.classification.__path__,
-            prefix="Orange.classification.",
-            onerror=lambda x: None)
-        for importer, modname, ispkg in classification_modules:
-            try:
-                module = pkgutil.importlib.import_module(modname)
-            except ImportError:
-                continue
-
-            for name, class_ in inspect.getmembers(module, inspect.isclass):
-                if (issubclass(class_, Learner) and
-                        not name.startswith('_') and
-                        'base' not in class_.__module__):
-                    yield class_
-
     def test_all_learners_accessible_in_Orange_classification_namespace(self):
-        for learner in self.all_learners():
+        for learner in all_learners():
             if not hasattr(Orange.classification, learner.__name__):
                 self.fail("%s is not visible in Orange.classification"
                           " namespace" % learner.__name__)
 
     def test_all_models_work_after_unpickling(self):
         datasets = [Table('iris'), Table('titanic')]
-        for learner in list(self.all_learners()):
+        for learner in list(all_learners()):
             try:
                 learner = learner()
             except Exception:
@@ -381,7 +429,7 @@ def test_all_models_work_after_unpickling(self):
                     % (learner.__class__.__name__, ds.name))
 
     def test_adequacy_all_learners(self):
-        for learner in self.all_learners():
+        for learner in all_learners():
             try:
                 learner = learner()
                 table = Table("housing")
@@ -391,7 +439,7 @@ def test_adequacy_all_learners(self):
                 continue
 
     def test_adequacy_all_learners_multiclass(self):
-        for learner in self.all_learners():
+        for learner in all_learners():
             try:
                 learner = learner()
                 table = Table(test_filename("datasets/test8.tab"))