Merge pull request #4780 from pavlin-policar/multinomial

logistic regression: sync defaults with scikit-learn

Author: PrimozGodec

Orange/classification/logistic_regression.py

@@ -36,7 +36,22 @@ class LogisticRegressionLearner(SklLearner, _FeatureScorerMixin):
 
     def __init__(self, penalty="l2", dual=False, tol=0.0001, C=1.0,
                  fit_intercept=True, intercept_scaling=1, class_weight=None,
-                 random_state=None, solver='liblinear', max_iter=100,
-                 multi_class='ovr', verbose=0, n_jobs=1, preprocessors=None):
+                 random_state=None, solver="auto", max_iter=100,
+                 multi_class="auto", verbose=0, n_jobs=1, preprocessors=None):
         super().__init__(preprocessors=preprocessors)
         self.params = vars()
+
+    def _initialize_wrapped(self):
+        params = self.params.copy()
+        # The default scikit-learn solver `lbfgs` (v0.22) does not support the
+        # l1 penalty.
+        solver, penalty = params.pop("solver"), params.get("penalty")
+        if solver == "auto":
+            if penalty == "l1":
+                solver = "liblinear"
+            else:
+                solver = "lbfgs"
+        params["solver"] = solver
+
+        return self.__wraps__(**params)
+

Orange/tests/test_evaluation_scoring.py

@@ -58,16 +58,12 @@ def setUpClass(cls):
     def test_precision_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
         res = TestOnTrainingData()(self.iris, [learner])
-        self.assertAlmostEqual(self.score(res, average='weighted')[0],
-                               0.96189, 5)
-        self.assertAlmostEqual(self.score(res, target=1)[0], 0.97826, 5)
-        self.assertAlmostEqual(self.score(res, target=1, average=None)[0],
-                               0.97826, 5)
-        self.assertAlmostEqual(self.score(res, target=1, average='weighted')[0],
-                               0.97826, 5)
-        self.assertAlmostEqual(self.score(res, target=0, average=None)[0], 1, 5)
-        self.assertAlmostEqual(self.score(res, target=2, average=None)[0],
-                               0.90741, 5)
+        self.assertGreater(self.score(res, average='weighted')[0], 0.95)
+        self.assertGreater(self.score(res, target=1)[0], 0.95)
+        self.assertGreater(self.score(res, target=1, average=None)[0], 0.95)
+        self.assertGreater(self.score(res, target=1, average='weighted')[0], 0.95)
+        self.assertGreater(self.score(res, target=0, average=None)[0], 0.99)
+        self.assertGreater(self.score(res, target=2, average=None)[0], 0.94)
 
     def test_precision_multiclass(self):
         results = Results(
@@ -117,15 +113,12 @@ def setUpClass(cls):
     def test_recall_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
         res = TestOnTrainingData()(self.iris, [learner])
-        self.assertAlmostEqual(self.score(res, average='weighted')[0], 0.96, 5)
-        self.assertAlmostEqual(self.score(res, target=1)[0], 0.9, 5)
-        self.assertAlmostEqual(self.score(res, target=1, average=None)[0],
-                               0.9, 5)
-        self.assertAlmostEqual(self.score(res, target=1, average='weighted')[0],
-                               0.9, 5)
-        self.assertAlmostEqual(self.score(res, target=0, average=None)[0], 1, 5)
-        self.assertAlmostEqual(self.score(res, target=2, average=None)[0],
-                               0.98, 5)
+        self.assertGreater(self.score(res, average='weighted')[0], 0.96)
+        self.assertGreater(self.score(res, target=1)[0], 0.9)
+        self.assertGreater(self.score(res, target=1, average=None)[0], 0.9)
+        self.assertGreater(self.score(res, target=1, average='weighted')[0], 0.9)
+        self.assertGreater(self.score(res, target=0, average=None)[0], 0.99)
+        self.assertGreater(self.score(res, target=2, average=None)[0], 0.97)
 
     def test_recall_multiclass(self):
         results = Results(
@@ -175,16 +168,12 @@ def setUpClass(cls):
     def test_recall_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
         res = TestOnTrainingData()(self.iris, [learner])
-        self.assertAlmostEqual(self.score(res, average='weighted')[0],
-                               0.959935, 5)
-        self.assertAlmostEqual(self.score(res, target=1)[0], 0.9375, 5)
-        self.assertAlmostEqual(self.score(res, target=1, average=None)[0],
-                               0.9375, 5)
-        self.assertAlmostEqual(self.score(res, target=1, average='weighted')[0],
-                               0.9375, 5)
-        self.assertAlmostEqual(self.score(res, target=0, average=None)[0], 1, 5)
-        self.assertAlmostEqual(self.score(res, target=2, average=None)[0],
-                               0.942307, 5)
+        self.assertGreater(self.score(res, average='weighted')[0], 0.95)
+        self.assertGreater(self.score(res, target=1)[0], 0.95)
+        self.assertGreater(self.score(res, target=1, average=None)[0], 0.95)
+        self.assertGreater(self.score(res, target=1, average='weighted')[0], 0.95)
+        self.assertGreater(self.score(res, target=0, average=None)[0], 0.99)
+        self.assertGreater(self.score(res, target=2, average=None)[0], 0.95)
 
     def test_F1_multiclass(self):
         results = Results(
@@ -377,16 +366,24 @@ def setUpClass(cls):
     def test_specificity_iris(self):
         learner = LogisticRegressionLearner(preprocessors=[])
         res = TestOnTrainingData()(self.iris, [learner])
-        self.assertAlmostEqual(self.score(res, average='weighted')[0],
-                               (1 + 0.99 + 0.95) / 3, 5)
-        self.assertAlmostEqual(self.score(res, target=1)[0], 99 / (99 + 1), 5)
-        self.assertAlmostEqual(self.score(res, target=1, average=None)[0],
-                               99 / (99 + 1), 5)
-        self.assertAlmostEqual(self.score(res, target=1, average='weighted')[0],
-                               99 / (99 + 1), 5)
-        self.assertAlmostEqual(self.score(res, target=0, average=None)[0], 1, 5)
-        self.assertAlmostEqual(self.score(res, target=2, average=None)[0],
-                               95 / (95 + 5), 5)
+        self.assertGreaterEqual(
+            self.score(res, average='weighted')[0], (1 + 0.99 + 0.95) / 3
+        )
+        self.assertGreaterEqual(
+            self.score(res, target=1)[0], 99 / (99 + 1)
+        )
+        self.assertGreaterEqual(
+            self.score(res, target=1, average=None)[0],  99 / (99 + 1)
+        )
+        self.assertGreaterEqual(
+            self.score(res, target=1, average='weighted')[0], 99 / (99 + 1)
+        )
+        self.assertGreaterEqual(
+            self.score(res, target=0, average=None)[0], 1
+        )
+        self.assertGreaterEqual(
+            self.score(res, target=2, average=None)[0], 95 / (95 + 5)
+        )
 
     def test_precision_multiclass(self):
         results = Results(

Orange/tests/test_logistic_regression.py

@@ -66,7 +66,7 @@ def test_probability(self):
     def test_learner_scorer(self):
         learner = LogisticRegressionLearner()
         scores = learner.score_data(self.heart_disease)
-        self.assertEqual('major vessels colored',
+        self.assertEqual('chest pain',
                          self.heart_disease.domain.attributes[np.argmax(scores)].name)
         self.assertEqual(scores.shape, (1, len(self.heart_disease.domain.attributes)))
 
@@ -89,13 +89,13 @@ def test_learner_scorer_multiclass(self):
         attr = self.zoo.domain.attributes
         learner = LogisticRegressionLearner()
         scores = learner.score_data(self.zoo)
-        self.assertEqual('aquatic', attr[np.argmax(scores[0])].name)  # amphibian
+        self.assertEqual('legs', attr[np.argmax(scores[0])].name)  # amphibian
         self.assertEqual('feathers', attr[np.argmax(scores[1])].name)  # bird
         self.assertEqual('fins', attr[np.argmax(scores[2])].name)  # fish
         self.assertEqual('legs', attr[np.argmax(scores[3])].name)  # insect
         self.assertEqual('backbone', attr[np.argmax(scores[4])].name)  # invertebrate
         self.assertEqual('milk', attr[np.argmax(scores[5])].name)  # mammal
-        self.assertEqual('hair', attr[np.argmax(scores[6])].name)  # reptile
+        self.assertEqual('aquatic', attr[np.argmax(scores[6])].name)  # reptile
         self.assertEqual(scores.shape,
                          (len(self.zoo.domain.class_var.values), len(attr)))
 
@@ -131,3 +131,23 @@ def test_sklearn_single_class(self):
         self.assertEqual(len(np.unique(t.Y)), 1)
         lr = sklearn.linear_model.LogisticRegression()
         self.assertRaises(ValueError, lr.fit, t.X, t.Y)
+
+    def test_auto_solver(self):
+        # These defaults are valid as of sklearn v0.23.0
+        # lbfgs is default for l2 penalty
+        lr = LogisticRegressionLearner(penalty="l2", solver="auto")
+        skl_clf = lr._initialize_wrapped()
+        self.assertEqual(skl_clf.solver, "lbfgs")
+        self.assertEqual(skl_clf.penalty, "l2")
+
+        # lbfgs is default for no penalty
+        lr = LogisticRegressionLearner(penalty=None, solver="auto")
+        skl_clf = lr._initialize_wrapped()
+        self.assertEqual(skl_clf.solver, "lbfgs")
+        self.assertEqual(skl_clf.penalty, None)
+
+        # liblinear is default for l2 penalty
+        lr = LogisticRegressionLearner(penalty="l1", solver="auto")
+        skl_clf = lr._initialize_wrapped()
+        self.assertEqual(skl_clf.solver, "liblinear")
+        self.assertEqual(skl_clf.penalty, "l1")

Orange/widgets/evaluate/tests/test_owtestandscore.py

@@ -382,26 +382,33 @@ def test_scores_log_reg_bad2(self):
         table_test = Table.from_list(
             self.scores_domain,
             list(zip(*(self.scores_table_values + [list("yynn")]))))
-        self.assertTupleEqual(self._test_scores(
-            table_train, table_test, LogisticRegressionLearner(),
-            OWTestAndScore.TestOnTest, None),
-                              (0, 0, 0, 0, 0))
+
+        lr = LogisticRegressionLearner()
+        np.testing.assert_almost_equal(
+            self._test_scores(
+                table_train, table_test, lr, OWTestAndScore.TestOnTest, None
+            ),
+            (0, 0.25, 0.2, 0.1666666, 0.25),
+        )
 
     def test_scores_log_reg_advanced(self):
         table_train = Table.from_list(
-            self.scores_domain, list(zip(
-                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], "yyynn"))
+            self.scores_domain,
+            list(zip([1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], "yyynn"))
         )
         table_test = Table.from_list(
-            self.scores_domain, list(zip(
-                [1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], "yynnn"))
+            self.scores_domain,
+            list(zip([1, 1, 1.23, 23.8, 5.], [1., 2., 3., 4., 3.], "yynnn"))
         )
 
+        lr = LogisticRegressionLearner()
+        np.testing.assert_
         np.testing.assert_almost_equal(
-            self._test_scores(table_train, table_test,
-                              LogisticRegressionLearner(),
-                              OWTestAndScore.TestOnTest, None),
-            (2 / 3, 0.8, 0.8, 13 / 15, 0.8))
+            self._test_scores(
+                table_train, table_test, lr, OWTestAndScore.TestOnTest, None
+            ),
+            (1, 0.8, 0.8, 13 / 15, 0.8)
+        )
 
     def test_scores_cross_validation(self):
         """

Orange/widgets/visualize/tests/test_ownomogram.py

@@ -97,7 +97,9 @@ def test_nomogram_nb_multiclass(self):
     def test_nomogram_lr_multiclass(self):
         """Check probabilities for logistic regression classifier for various
         values of classes and radio buttons for multiclass data"""
-        cls = LogisticRegressionLearner()(self.lenses)
+        cls = LogisticRegressionLearner(
+            multi_class="ovr", solver="liblinear"
+        )(self.lenses)
         self._test_helper(cls, [9, 45, 52])
 
     def test_nomogram_with_instance_nb(self):