Merge pull request #46 from adolfo-ab/fairness-property-testing

test: additional unit tests for SPD and DIR metrics

Author: adolfo-ab

pyproject.toml

@@ -17,6 +17,8 @@ dependencies = [
     "h5py>=3.13.0,<4",
     "scikit-learn",
     "aif360",
+    "hypothesis>=6.136.2",
+    "pytest>=8.4.1",
 ]
 
 [project.optional-dependencies]

src/core/metrics/fairness/fairness_metrics_utils.py

@@ -2,9 +2,11 @@
 import numpy as np
 from sklearn.metrics import confusion_matrix
 
+
 def filter_rows_by_inputs(data: np.ndarray, filter_func: Callable[[np.ndarray], bool]):
     return data[np.apply_along_axis(filter_func, 1, data)]
 
+
 def calculate_confusion_matrix(test: np.array, truth: np.array, positive_class: int) -> dict:
     # cast test and truth to int
     test = test.astype(int)
@@ -16,3 +18,14 @@ def calculate_confusion_matrix(test: np.array, truth: np.array, positive_class:
     fp = cm[1, 0]
     tn = cm[1, 1]
     return {"tp": tp, "tn": tn, "fp": fp, "fn": fn}
+
+
+def validate_fairness_groups(privileged: np.ndarray, unprivileged: np.ndarray) -> None:
+    empty_groups = []
+    if len(privileged) == 0:
+        empty_groups.append("privileged")
+    if len(unprivileged) == 0:
+        empty_groups.append("unprivileged")
+
+    if empty_groups:
+        raise ValueError(f"Arrays cannot be empty for the following groups: {', '.join(empty_groups)}")

src/core/metrics/fairness/group/disparate_impact_ratio.py

@@ -4,18 +4,22 @@
 import numpy as np
 from sklearn.base import ClassifierMixin
 
+from src.core.metrics.fairness.fairness_metrics_utils import validate_fairness_groups
+
+
 class DisparateImpactRatio:
     """
     Calculate disparate impact ratio (DIR).
     """
+
     @staticmethod
     def calculate_model(
         samples: np.ndarray,
         model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
-        favorable_output: np.ndarray
-        ) -> float:
+        favorable_output: np.ndarray,
+    ) -> float:
         """
         Calculate disparate impact ratio (DIR) for model outputs.
         :param samples a NumPy array of inputs to be used for testing fairness
@@ -34,17 +38,17 @@ def calculate_model(
 
     @staticmethod
     def calculate(
-        privileged: Union[int, np.ndarray],
-        unprivileged: Union[int, np.ndarray],
-        favorable_output: int
-        ) -> float:
+        privileged: Union[int, np.ndarray], unprivileged: Union[int, np.ndarray], favorable_output: int
+    ) -> float:
         """
         Calculate disparate impact ratio (DIR) when the labels are pre-calculated.
         :param privileged a NumPy array with the privileged groups
         :param unprivileged a NumPy array with the unprivileged groups
         :param favorableOutput an output that is considered favorable / desirable
         return DIR, between 0 and 1
         """
+        validate_fairness_groups(privileged=privileged, unprivileged=unprivileged)
+
         probability_privileged = np.sum(privileged[:, -1] == favorable_output) / len(privileged)
         probability_unprivileged = np.sum(unprivileged[:, -1] == favorable_output) / len(unprivileged)
         return probability_unprivileged / probability_privileged

src/core/metrics/fairness/group/group_statistical_parity_difference.py

@@ -4,18 +4,22 @@
 import numpy as np
 from sklearn.base import ClassifierMixin
 
+from src.core.metrics.fairness.fairness_metrics_utils import validate_fairness_groups
+
+
 class GroupStatisticalParityDifference:
     """
     Calculate group statistical parity difference (SPD).
     """
+
     @staticmethod
     def calculate_model(
         samples: np.ndarray,
         model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
         favorable_output,
-        ) -> float:
+    ) -> float:
         """
         Calculate group statistical parity difference (SPD) for model outputs.
         :param samples a NumPy array of inputs to be used for testing fairness
@@ -37,14 +41,16 @@ def calculate(
         privileged,
         unprivileged,
         favorable_output,
-        ) -> float:
+    ) -> float:
         """
         Calculate statistical/demographic parity difference (SPD) when the labels are pre-calculated.
         :param priviledged numPy array with the privileged groups
         :param unpriviledged numPy array with the unpriviledged groups
         :param favorableOutput an output that is considered favorable / desirable
-        return SPD, between 0 and 1
+        return SPD, between -1 and 1
         """
+        validate_fairness_groups(privileged=privileged, unprivileged=unprivileged)
+
         probability_privileged = np.sum(privileged[:, -1] == favorable_output) / len(privileged)
         probability_unprivileged = np.sum(unprivileged[:, -1] == favorable_output) / len(unprivileged)
         return probability_unprivileged - probability_privileged