Merge pull request #45 from christinaexyou/patch-fairness-issues

fix: Fairness metrics and tests

Author: christinaexyou

src/core/metrics/fairness/fairness_metrics_utils.py

@@ -1,12 +1,18 @@
 from typing import Callable
 import numpy as np
+from sklearn.metrics import confusion_matrix
 
 def filter_rows_by_inputs(data: np.ndarray, filter_func: Callable[[np.ndarray], bool]):
     return data[np.apply_along_axis(filter_func, 1, data)]
 
 def calculate_confusion_matrix(test: np.array, truth: np.array, positive_class: int) -> dict:
-    tp = np.sum((test == positive_class) & (truth == positive_class))
-    tn = np.sum((test != positive_class) & (truth != positive_class))
-    fp = np.sum((test == positive_class) & (truth != positive_class))
-    fn = np.sum((test != positive_class) & (truth == positive_class))
-    return {"tp": tp, "tn": tn, "fp": fp, "fn": fn}
+    # cast test and truth to int
+    test = test.astype(int)
+    truth = truth.astype(int)
+    # calculate confusion matrix
+    cm = confusion_matrix(truth, test, labels=[positive_class, 1 - positive_class])
+    tp = cm[0, 0]
+    fn = cm[0, 1]
+    fp = cm[1, 0]
+    tn = cm[1, 1]
+    return {"tp": tp, "tn": tn, "fp": fp, "fn": fn}

src/core/metrics/fairness/group/disparate_impact_ratio.py

@@ -1,7 +1,8 @@
 # pylint: disable=line-too-long
-from typing import List, Any, Union
+from typing import List, Union
 
 import numpy as np
+from sklearn.base import ClassifierMixin
 
 class DisparateImpactRatio:
     """
@@ -10,7 +11,7 @@ class DisparateImpactRatio:
     @staticmethod
     def calculate_model(
         samples: np.ndarray,
-        model: Any,
+        model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
         favorable_output: np.ndarray

src/core/metrics/fairness/group/group_average_odds_difference.py

@@ -1,7 +1,8 @@
 # pylint: disable=line-too-long, too-many-arguments
-from typing import List, Any
+from typing import List
 
 import numpy as np
+from sklearn.base import ClassifierMixin
 
 from src.core.metrics.fairness.fairness_metrics_utils import filter_rows_by_inputs, calculate_confusion_matrix
 
@@ -12,7 +13,7 @@ class GroupAverageOddsDifference:
     @ staticmethod
     def calculate_model(
         samples: np.ndarray,
-        model: Any,
+        model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
         postive_class: List[int],

src/core/metrics/fairness/group/group_average_predictive_value_difference.py

@@ -1,7 +1,8 @@
 # pylint: disable=line-too-long, too-many-arguments
-from typing import List, Any
+from typing import List
 
 import numpy as np
+from sklearn.base import ClassifierMixin
 
 from src.core.metrics.fairness.fairness_metrics_utils import filter_rows_by_inputs, calculate_confusion_matrix
 
@@ -12,7 +13,7 @@ class GroupAveragePredictiveValueDifference:
     @staticmethod
     def calculate_model(
         samples: np.ndarray,
-        model: Any,
+        model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
         positive_class: int,

src/core/metrics/fairness/group/group_statistical_parity_difference.py

@@ -2,6 +2,7 @@
 from typing import List
 
 import numpy as np
+from sklearn.base import ClassifierMixin
 
 class GroupStatisticalParityDifference:
     """
@@ -10,7 +11,7 @@ class GroupStatisticalParityDifference:
     @staticmethod
     def calculate_model(
         samples: np.ndarray,
-        model,
+        model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
         favorable_output,

src/core/metrics/fairness/individual/individual_consistency.py

@@ -2,6 +2,7 @@
 from typing import Any
 
 import numpy as np
+from sklearn.base import ClassifierMixin
 
 class IndividualConsistency:
     """
@@ -15,21 +16,23 @@ class IndividualConsistency:
     def calculate(
         proximity_function: Any,
         samples: np.ndarray,
-        prediction_provider: Any
+        model: ClassifierMixin
     ) -> float:
         """
         Calculate individual fairness.
         :param proximity_function: a function that finds the top k similar inputs, given a reference input and a list of inputs
-        :param samples a list of inputs to be tested for consistency
-        :param prediction_provider the model under inspection
+        :param samples: a list of inputs to be tested for consistency
+        :param model: the model under inspection
         return the consistency measure
         """
         consistency =  1
         for sample in samples:
-            prediction_outputs = prediction_provider.predict(sample)
+            prediction_outputs = model.predict(sample)
+            if len(prediction_outputs) == 0:
+                raise ValueError("Model output cannot be empty.")
             prediction_output = prediction_outputs[0]
             neighbors = proximity_function(sample, samples)
-            neighbors_outputs = prediction_provider.predict(neighbors)
+            neighbors_outputs = model.predict(neighbors)
             for output in prediction_outputs:
                 for neighbor_output in neighbors_outputs:
                     if neighbor_output != output: