raise ValueError when privileged/unprivileged arrays are empty

Author: adolfo-ab

src/core/metrics/fairness/fairness_metrics_utils.py

@@ -2,9 +2,11 @@
 import numpy as np
 from sklearn.metrics import confusion_matrix
 
+
 def filter_rows_by_inputs(data: np.ndarray, filter_func: Callable[[np.ndarray], bool]):
     return data[np.apply_along_axis(filter_func, 1, data)]
 
+
 def calculate_confusion_matrix(test: np.array, truth: np.array, positive_class: int) -> dict:
     # cast test and truth to int
     test = test.astype(int)
@@ -16,3 +18,14 @@ def calculate_confusion_matrix(test: np.array, truth: np.array, positive_class:
     fp = cm[1, 0]
     tn = cm[1, 1]
     return {"tp": tp, "tn": tn, "fp": fp, "fn": fn}
+
+
+def validate_fairness_groups(privileged: np.ndarray, unprivileged: np.ndarray) -> None:
+    empty_groups = []
+    if len(privileged) == 0:
+        empty_groups.append("privileged")
+    if len(unprivileged) == 0:
+        empty_groups.append("unprivileged")
+
+    if empty_groups:
+        raise ValueError(f"Arrays cannot be empty for the following groups: {', '.join(empty_groups)}")

src/core/metrics/fairness/group/disparate_impact_ratio.py

@@ -4,18 +4,22 @@
 import numpy as np
 from sklearn.base import ClassifierMixin
 
+from src.core.metrics.fairness.fairness_metrics_utils import validate_fairness_groups
+
+
 class DisparateImpactRatio:
     """
     Calculate disparate impact ratio (DIR).
     """
+
     @staticmethod
     def calculate_model(
         samples: np.ndarray,
         model: ClassifierMixin,
         privilege_columns: List[int],
         privilege_values: List[int],
-        favorable_output: np.ndarray
-        ) -> float:
+        favorable_output: np.ndarray,
+    ) -> float:
         """
         Calculate disparate impact ratio (DIR) for model outputs.
         :param samples a NumPy array of inputs to be used for testing fairness
@@ -34,17 +38,17 @@ def calculate_model(
 
     @staticmethod
     def calculate(
-        privileged: Union[int, np.ndarray],
-        unprivileged: Union[int, np.ndarray],
-        favorable_output: int
-        ) -> float:
+        privileged: Union[int, np.ndarray], unprivileged: Union[int, np.ndarray], favorable_output: int
+    ) -> float:
         """
         Calculate disparate impact ratio (DIR) when the labels are pre-calculated.
         :param privileged a NumPy array with the privileged groups
         :param unprivileged a NumPy array with the unprivileged groups
         :param favorableOutput an output that is considered favorable / desirable
         return DIR, between 0 and 1
         """
+        validate_fairness_groups(privileged=privileged, unprivileged=unprivileged)
+
         probability_privileged = np.sum(privileged[:, -1] == favorable_output) / len(privileged)
         probability_unprivileged = np.sum(unprivileged[:, -1] == favorable_output) / len(unprivileged)
         return probability_unprivileged / probability_privileged

src/core/metrics/fairness/group/group_statistical_parity_difference.py

@@ -4,6 +4,8 @@
 import numpy as np
 from sklearn.base import ClassifierMixin
 
+from src.core.metrics.fairness.fairness_metrics_utils import validate_fairness_groups
+
 
 class GroupStatisticalParityDifference:
     """
@@ -47,6 +49,8 @@ def calculate(
         :param favorableOutput an output that is considered favorable / desirable
         return SPD, between -1 and 1
         """
+        validate_fairness_groups(privileged=privileged, unprivileged=unprivileged)
+
         probability_privileged = np.sum(privileged[:, -1] == favorable_output) / len(privileged)
         probability_unprivileged = np.sum(unprivileged[:, -1] == favorable_output) / len(unprivileged)
         return probability_unprivileged - probability_privileged

tests/metrics/test_fairness.py

@@ -1,6 +1,6 @@
 # pylint: disable=line-too-long, missing-function-docstring
-import math
 
+import pytest
 from pytest import approx
 import numpy as np
 import pandas as pd
@@ -234,9 +234,8 @@ def test_dir_empty_dataframe(self):
         empty_df = pd.DataFrame(columns=df.columns)
         privileged, unprivileged = get_privileged_unprivileged_split(df=empty_df)
 
-        score = DisparateImpactRatio.calculate(privileged=privileged, unprivileged=unprivileged, favorable_output=1)
-
-        assert math.isnan(score)
+        with pytest.raises(ValueError):
+            DisparateImpactRatio.calculate(privileged=privileged, unprivileged=unprivileged, favorable_output=1)
 
 
 class TestGroupStatisticalParityDifference:
@@ -305,11 +304,10 @@ def test_spd_empty_dataframe(self):
         empty_df = pd.DataFrame(columns=df.columns)
         privileged, unprivileged = get_privileged_unprivileged_split(df=empty_df)
 
-        score = GroupStatisticalParityDifference.calculate(
-            privileged=privileged, unprivileged=unprivileged, favorable_output=1
-        )
-
-        assert math.isnan(score)
+        with pytest.raises(ValueError):
+            GroupStatisticalParityDifference.calculate(
+                privileged=privileged, unprivileged=unprivileged, favorable_output=1
+            )
 
 
 def test_average_odds_difference():