Implemented specificity

Author: PrimozGodec

Orange/evaluation/scoring.py

@@ -14,6 +14,7 @@
 
 import numpy as np
 import sklearn.metrics as skl_metrics
+from sklearn.metrics import confusion_matrix
 
 from Orange.data import DiscreteVariable, ContinuousVariable
 from Orange.misc.wrapper_meta import WrapperMeta
@@ -281,6 +282,57 @@ def compute_score(self, results, eps=1e-15, normalize=True,
             dtype=np.float64, count=len(results.probabilities))
 
 
+class Specificity(ClassificationScore):
+    is_binary = True
+
+    def calculate_weights(self, results):
+        classes, counts = np.unique(results.actual, return_counts=True)
+        n = np.array(results.actual).shape[0]
+        return counts / n, classes
+
+    @staticmethod
+    def specificity(y_true, y_pred):
+        tn, fp, fn, tp = confusion_matrix(y_true, y_pred).ravel()
+        return tn / (tn + fp)
+
+    def single_class_specificity(self, results, target):
+        y_true = (np.array(results.actual) == target).astype(int)
+        return np.fromiter(
+            (self.specificity(y_true,
+                              np.array(predicted == target, dtype=int))
+             for predicted in results.predicted),
+            dtype=np.float64, count=len(results.predicted))
+
+    def multi_class_specificity(self, results):
+        weights, classes = self.calculate_weights(results)
+        scores = np.array([self.single_class_specificity(results, class_)
+                              for class_ in classes])
+        return np.sum(scores.T * weights, axis=1)
+
+    def compute_score(self, results, target=None, average="binary"):
+        domain = results.domain
+        n_classes = len(domain.class_var.values)
+
+        if n_classes == 0:
+            raise ValueError("Class variable has less than one values")
+        else:
+            if target is None:
+                if average == "weighted":
+                    return self.multi_class_specificity(results)
+                else:  # average is binary
+                    if n_classes != 2:
+                        raise ValueError(
+                            "Binary averaging needs two classes in data: "
+                            "specify target class or use "
+                            "weighted averaging.")
+                    return self.single_class_specificity(results, 1)
+            elif target is not None:
+                return self.single_class_specificity(results, target)
+            else:
+                raise ValueError(
+                    "Wrong parameters: For averaging select one of the "
+                    "following values: ('weighted', 'binary')")
+
 # Regression scores
 
 class MSE(RegressionScore):

Orange/tests/test_evaluation_scoring.py

@@ -10,6 +10,7 @@
                                   MajorityLearner
 from Orange.evaluation import AUC, CA, Results, Recall, \
     Precision, TestOnTrainingData, scoring, LogLoss, F1, CrossValidation
+from Orange.evaluation.scoring import Specificity
 from Orange.preprocess import discretize, Discretize
 from Orange.tests import test_filename
 
@@ -367,6 +368,62 @@ def test_log_loss_calc(self):
         self.assertAlmostEqual(ll_calc, ll_orange[0])
 
 
+class TestSpecificity(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        cls.iris = Table('iris')
+        cls.score = Specificity()
+
+    def test_specificity_iris(self):
+        learner = LogisticRegressionLearner(preprocessors=[])
+        res = TestOnTrainingData()(self.iris, [learner])
+        self.assertAlmostEqual(self.score(res, average='weighted')[0],
+                               (1 + 0.99 + 0.95) / 3, 5)
+        self.assertAlmostEqual(self.score(res, target=1)[0], 99 / (99 + 1), 5)
+        self.assertAlmostEqual(self.score(res, target=1, average=None)[0],
+                               99 / (99 + 1), 5)
+        self.assertAlmostEqual(self.score(res, target=1, average='weighted')[0],
+                               99 / (99 + 1), 5)
+        self.assertAlmostEqual(self.score(res, target=0, average=None)[0], 1, 5)
+        self.assertAlmostEqual(self.score(res, target=2, average=None)[0],
+                               95 / (95 + 5), 5)
+
+    def test_precision_multiclass(self):
+        results = Results(
+            domain=Domain([], DiscreteVariable(name="y", values="01234")),
+            actual=[0, 4, 4, 1, 2, 0, 1, 2, 3, 2])
+        results.predicted = np.array([[0, 4, 4, 1, 2, 0, 1, 2, 3, 2],
+                                      [0, 1, 4, 1, 1, 0, 0, 2, 3, 1]])
+        res = self.score(results, average='weighted')
+        self.assertEqual(res[0], 1.)
+        self.assertAlmostEqual(res[1], 0.9, 5)
+
+        for target, prob in ((0, 7 / 8),
+                             (1, 5 / 8),
+                             (2, 1),
+                             (3, 1),
+                             (4, 1)):
+            res = self.score(results, target=target, average=None)
+            self.assertEqual(res[0], 1.)
+            self.assertEqual(res[1], prob)
+
+    def test_precision_binary(self):
+        results = Results(
+            domain=Domain([], DiscreteVariable(name="y", values="01")),
+            actual=[0, 1, 1, 1, 0, 0, 1, 0, 0, 1])
+        results.predicted = np.array([[0, 1, 1, 1, 0, 0, 1, 0, 0, 1],
+                                      [0, 1, 1, 1, 0, 0, 1, 1, 1, 0]])
+        res = self.score(results)
+        self.assertEqual(res[0], 1.)
+        self.assertAlmostEqual(res[1], 3 / 5)
+        res_target = self.score(results, target=1)
+        self.assertEqual(res[0], res_target[0])
+        self.assertEqual(res[1], res_target[1])
+        res_target = self.score(results, target=0)
+        self.assertEqual(res_target[0], 1.)
+        self.assertAlmostEqual(res_target[1], 4 / 5)
+
+
 if __name__ == '__main__':
     unittest.main()
     del TestScoreMetaType