.

Author: ahn1340

autosklearn/metrics/__init__.py

@@ -185,35 +185,49 @@ def make_scorer(name, score_func, optimum=1, greater_is_better=True,
 # Standard regression scores
 r2 = make_scorer('r2', sklearn.metrics.r2_score)
 mean_squared_error = make_scorer('mean_squared_error',
-                                 sklearn.metrics.mean_squared_error, optimum=0,
+                                 sklearn.metrics.mean_squared_error,
+                                 optimum=0,
                                  greater_is_better=False)
 mean_absolute_error = make_scorer('mean_absolute_error',
                                   sklearn.metrics.mean_absolute_error,
-                                  optimum=0, greater_is_better=False)
+                                  optimum=0,
+                                  greater_is_better=False)
 median_absolute_error = make_scorer('median_absolute_error',
                                     sklearn.metrics.median_absolute_error,
-                                    optimum=0, greater_is_better=False)
+                                    optimum=0,
+                                    greater_is_better=False)
 
 # Standard Classification Scores
-accuracy = make_scorer('accuracy', sklearn.metrics.accuracy_score)
+accuracy = make_scorer('accuracy',
+                       sklearn.metrics.accuracy_score)
 balanced_accuracy = make_scorer('balanced_accuracy',
                                 classification_metrics.balanced_accuracy)
-f1 = make_scorer('f1', sklearn.metrics.f1_score)
+f1 = make_scorer('f1',
+                 sklearn.metrics.f1_score)
 
 # Score functions that need decision values
-roc_auc = make_scorer('roc_auc', sklearn.metrics.roc_auc_score,
-                      greater_is_better=True, needs_threshold=True)
+roc_auc = make_scorer('roc_auc',
+                      sklearn.metrics.roc_auc_score,
+                      greater_is_better=True,
+                      needs_threshold=True)
 average_precision = make_scorer('average_precision',
                                 sklearn.metrics.average_precision_score,
                                 needs_threshold=True)
-precision = make_scorer('precision', sklearn.metrics.precision_score)
-recall = make_scorer('recall', sklearn.metrics.recall_score)
+precision = make_scorer('precision',
+                        sklearn.metrics.precision_score)
+recall = make_scorer('recall',
+                     sklearn.metrics.recall_score)
 
 # Score function for probabilistic classification
-log_loss = make_scorer('log_loss', sklearn.metrics.log_loss, optimum=0,
-                       greater_is_better=False, needs_proba=True)
-pac_score = make_scorer('pac_score', classification_metrics.pac_score,
-                        greater_is_better=True, needs_proba=True)
+log_loss = make_scorer('log_loss',
+                       sklearn.metrics.log_loss,
+                       optimum=0,
+                       greater_is_better=False,
+                       needs_proba=True)
+pac_score = make_scorer('pac_score',
+                        classification_metrics.pac_score,
+                        greater_is_better=True,
+                        needs_proba=True)
 # TODO what about mathews correlation coefficient etc?
 
 

test/test_metric/test_metrics.py

@@ -317,12 +317,7 @@ def test_regression_all(self):
         for metric, scorer in autosklearn.metrics.REGRESSION_METRICS.items():
             y_true = np.array([1, 2, 3, 4])
             y_pred = y_true.copy()
-
-            # the best possible score of r2 loss is 1.
-            if metric == 'r2':
-                previous_score = 1
-            else:
-                previous_score = 0
+            previous_score = scorer._optimum
             current_score = scorer(y_true, y_pred)
             self.assertAlmostEqual(current_score, previous_score)
 
@@ -346,18 +341,17 @@ def test_classification_binary(self):
         for metric, scorer in autosklearn.metrics.CLASSIFICATION_METRICS.items():
             # Skip functions not applicable for binary classification.
             # TODO: Average precision should work for binary classification,
-            # TODO: but its behavior is not right.
+            # TODO: but its behavior is not right. When y_pred is completely
+            # TODO: wrong, it does return 0.5, but when it is not completely
+            # TODO: wrong, it returns value smaller than 0.5.
             if metric in ['average_precision', 'pac_score',
                           'precision_samples', 'recall_samples', 'f1_samples']:
                 continue
 
             y_true = np.array([1.0, 1.0, 1.0, 0.0, 0.0, 0.0])
             y_pred = np.array([[0.0, 1.0], [0.0, 1.0], [0.0, 1.0], [1.0, 0.0],
                                 [1.0, 0.0], [1.0, 0.0]])
-            if metric is 'log_loss':
-                previous_score = 0      # the best value for log loss is 0.
-            else:
-                previous_score = 1     # the best value for other losses is 1.
+            previous_score = scorer._optimum
             current_score = scorer(y_true, y_pred)
             self.assertAlmostEqual(current_score, previous_score)
 
@@ -390,10 +384,7 @@ def test_classification_multiclass(self):
             y_true = np.array([0.0, 0.0, 1.0, 1.0, 2.0])
             y_pred = np.array([[1.0, 0.0, 0.0], [1.0, 0.0, 0.0],
                             [0.0, 1.0, 0.0], [0.0, 1.0, 0.0], [0.0, 0.0, 1.0]])
-            if metric is 'log_loss': # the best possible score for log_loss is 0.
-                previous_score = 0
-            else:
-                previous_score = 1 # the best value for other losses is 1.
+            previous_score = scorer._optimum
             current_score = scorer(y_true, y_pred)
             self.assertAlmostEqual(current_score, previous_score)
 
@@ -424,7 +415,7 @@ def test_classification_multilabel(self):
                 continue
             y_true = np.array([[1, 0, 0], [1, 1, 0], [0, 1, 1], [1, 1, 1]])
             y_pred = y_true.copy()
-            previous_score = 1
+            previous_score = scorer._optimum
             current_score = scorer(y_true, y_pred)
             self.assertAlmostEqual(current_score, previous_score)