Merge pull request #1980 from bluewave-labs/develop

Develop -> Master (26 Aug)

Author: gorkem-bwl

.gitignore

@@ -13,3 +13,4 @@ venv/
 express.log
 fastapi.log
 react.log
+.idea/

BiasAndFairnessModule/.gitignore

@@ -1,5 +1,2 @@
-<<<<<<< HEAD
 *.zip
-=======
 /artifacts
->>>>>>> upstream/develop

BiasAndFairnessModule/tests/base_test_metrics.py

@@ -0,0 +1,59 @@
+import unittest
+
+import numpy as np
+
+
+class BaseMetricsTestCase(unittest.TestCase):
+    """
+    Shared base class for fairness metric tests.
+
+    Generates a small, reproducible synthetic dataset with bias across a
+    protected attribute. Exposes:
+      - self.n_samples
+      - self.protected_attributes (group 0/1)
+      - self.y_true (0/1)
+      - self.y_pred (0/1)
+      - self.y_scores ([0, 1])
+      - self.legitimate_attributes (categorical strata for conditional metrics)
+    """
+
+    def setUp(self):
+        # Global seed for reproducibility across all tests inheriting this base
+        np.random.seed(42)
+
+        # Match the provided generation spec exactly for deterministic results
+        self.n_samples = 500
+
+        # Protected attribute with class imbalance
+        self.protected_attributes = np.random.choice(
+            [0, 1], size=self.n_samples, p=[0.7, 0.3]
+        ).astype(int)
+
+        # Ground truth labels
+        self.y_true = np.random.choice(
+            [0, 1], size=self.n_samples, p=[0.6, 0.4]
+        ).astype(int)
+
+        # Biased predictions (group 1 has higher positive rate)
+        self.y_pred = np.zeros(self.n_samples, dtype=int)
+        for i in range(self.n_samples):
+            if self.protected_attributes[i] == 0:
+                self.y_pred[i] = np.random.choice([0, 1], p=[0.8, 0.2])
+            else:
+                self.y_pred[i] = np.random.choice([0, 1], p=[0.6, 0.4])
+
+        # Continuous scores (clipped to [0,1] while preserving bias pattern)
+        self.y_scores = (
+            np.random.random(self.n_samples) * 0.8
+        )  # Scale down initial values
+        self.y_scores[
+            self.protected_attributes == 1
+        ] += 0.2  # Add bias while staying in [0,1]
+
+        # Legitimate attribute for conditional metrics (e.g., conditional statistical parity)
+        self.legitimate_attributes = np.random.choice(
+            [0, 1, 2], size=self.n_samples, p=[0.5, 0.3, 0.2]
+        ).astype(int)
+
+
+# Intentionally no tests here; concrete test classes should inherit from BaseMetricsTestCase.

BiasAndFairnessModule/tests/test_balance_negative_class.py

@@ -0,0 +1,23 @@
+import unittest
+
+from src.metrics import balance_negative_class
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestBalanceNegativeClass(BaseMetricsTestCase):
+    def test_balance_negative_class_ratio_and_difference(self):
+        metric_frame = balance_negative_class(
+            y_true=self.y_true,
+            y_pred_proba=self.y_scores,
+            protected_attributes=self.protected_attributes,
+        )
+
+        ratio = float(metric_frame.ratio(method="between_groups"))
+        difference = float(metric_frame.difference(method="between_groups"))
+
+        self.assertAlmostEqual(ratio, 0.684072790, places=9)
+        self.assertAlmostEqual(difference, 0.185452927, places=9)
+
+
+if __name__ == "__main__":
+    unittest.main()

BiasAndFairnessModule/tests/test_balance_positive_class.py

@@ -0,0 +1,23 @@
+import unittest
+
+from src.metrics import balance_positive_class
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestBalancePositiveClass(BaseMetricsTestCase):
+    def test_balance_positive_class_ratio_and_difference(self):
+        metric_frame = balance_positive_class(
+            y_true=self.y_true,
+            y_pred_proba=self.y_scores,
+            protected_attributes=self.protected_attributes,
+        )
+
+        ratio = float(metric_frame.ratio(method="between_groups"))
+        difference = float(metric_frame.difference(method="between_groups"))
+
+        self.assertAlmostEqual(ratio, 0.648702328, places=9)
+        self.assertAlmostEqual(difference, 0.213191682, places=9)
+
+
+if __name__ == "__main__":
+    unittest.main()

BiasAndFairnessModule/tests/test_calibration.py

@@ -0,0 +1,23 @@
+import unittest
+
+from src.metrics import calibration
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestCalibrationMetric(BaseMetricsTestCase):
+    def test_calibration_ratio_and_difference(self):
+        metric_frame = calibration(
+            y_true=self.y_true,
+            y_pred_proba=self.y_scores,
+            protected_attributes=self.protected_attributes,
+        )
+
+        ratio = float(metric_frame.ratio(method="between_groups"))
+        difference = float(metric_frame.difference(method="between_groups"))
+
+        self.assertAlmostEqual(ratio, 0.8464818194, places=10)
+        self.assertAlmostEqual(difference, 0.051970213833, places=12)
+
+
+if __name__ == "__main__":
+    unittest.main()

BiasAndFairnessModule/tests/test_conditional_statistical_parity.py

@@ -0,0 +1,62 @@
+import unittest
+
+from src.metrics import conditional_statistical_parity
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestConditionalStatisticalParity(BaseMetricsTestCase):
+    def test_conditional_statistical_parity_values(self):
+        result = conditional_statistical_parity(
+            y_pred=self.y_pred,
+            protected_attributes=self.protected_attributes,
+            legitimate_attributes=self.legitimate_attributes,
+        )
+
+        # Expected values as specified
+        expected = [
+            {
+                "stratum": "0",
+                "group_selection_rates": {"0": 0.19886363636363635, "1": 0.48},
+                "disparity": 0.28113636363636363,
+            },
+            {
+                "stratum": "2",
+                "group_selection_rates": {"0": 0.2753623188405797, "1": 0.5625},
+                "disparity": 0.2871376811594203,
+            },
+            {
+                "stratum": "1",
+                "group_selection_rates": {
+                    "0": 0.2376237623762376,
+                    "1": 0.44680851063829785,
+                },
+                "disparity": 0.20918474826206024,
+            },
+        ]
+
+        # Compare ignoring list order by indexing by stratum
+        result_by_stratum = {entry["stratum"]: entry for entry in result}
+        expected_by_stratum = {entry["stratum"]: entry for entry in expected}
+
+        self.assertEqual(set(result_by_stratum.keys()), set(expected_by_stratum.keys()))
+
+        # Tolerance for floating point comparisons
+        for stratum, expected_entry in expected_by_stratum.items():
+            self.assertIn(stratum, result_by_stratum)
+            got_entry = result_by_stratum[stratum]
+
+            # Check group_selection_rates
+            exp_rates = expected_entry["group_selection_rates"]
+            got_rates = got_entry["group_selection_rates"]
+            self.assertEqual(set(exp_rates.keys()), set(got_rates.keys()))
+            for group_key, exp_val in exp_rates.items():
+                self.assertAlmostEqual(got_rates[group_key], exp_val, places=12)
+
+            # Check disparity
+            self.assertAlmostEqual(
+                got_entry["disparity"], expected_entry["disparity"], places=12
+            )
+
+
+if __name__ == "__main__":
+    unittest.main()

BiasAndFairnessModule/tests/test_conditional_use_accuracy_equality.py

@@ -0,0 +1,29 @@
+import unittest
+
+from src.metrics import conditional_use_accuracy_equality
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestConditionalUseAccuracyEquality(BaseMetricsTestCase):
+    def test_npv_and_ppv_ratio_and_difference(self):
+        result = conditional_use_accuracy_equality(
+            y_true=self.y_true,
+            y_pred=self.y_pred,
+            protected_attributes=self.protected_attributes,
+        )
+
+        # NPV checks
+        npv_ratio = float(result.npv.ratio(method="between_groups"))
+        npv_diff = float(result.npv.difference(method="between_groups"))
+        self.assertAlmostEqual(npv_ratio, 0.943449929, places=9)
+        self.assertAlmostEqual(npv_diff, 0.035660306, places=9)
+
+        # PPV checks
+        ppv_ratio = float(result.ppv.ratio(method="between_groups"))
+        ppv_diff = float(result.ppv.difference(method="between_groups"))
+        self.assertAlmostEqual(ppv_ratio, 0.78, places=9)
+        self.assertAlmostEqual(ppv_diff, 0.078974359, places=9)
+
+
+if __name__ == "__main__":
+    unittest.main()

BiasAndFairnessModule/tests/test_demographic_parity.py

@@ -0,0 +1,23 @@
+import unittest
+
+import numpy as np
+
+from src.metrics import demographic_parity
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestDemographicParity(BaseMetricsTestCase):
+    def test_value_in_range(self):
+        value = demographic_parity(self.y_true, self.y_pred, self.protected_attributes)
+        self.assertTrue(0.0 <= value <= 1.0)
+
+    def test_value_close_to_expected(self):
+        value = demographic_parity(self.y_true, self.y_pred, self.protected_attributes)
+        self.assertTrue(
+            np.isclose(value, 0.26157, atol=0.05),
+            msg=f"demographic_parity={value}",
+        )
+
+
+if __name__ == "__main__":
+    unittest.main()

BiasAndFairnessModule/tests/test_equal_selection_parity.py

@@ -0,0 +1,27 @@
+import unittest
+
+import numpy as np
+
+from src.metrics import equal_selection_parity
+from tests.base_test_metrics import BaseMetricsTestCase
+
+
+class TestEqualSelectionParity(BaseMetricsTestCase):
+    def test_output_and_expected_counts(self):
+        result = equal_selection_parity(
+            self.y_true, self.y_pred, self.protected_attributes
+        )
+
+        # Output format: keys are np.int64 group labels, values are Python ints
+        self.assertIn(np.int64(0), result)
+        self.assertIn(np.int64(1), result)
+        self.assertIsInstance(result[np.int64(0)], int)
+        self.assertIsInstance(result[np.int64(1)], int)
+
+        # Exact expected counts
+        self.assertEqual(result[np.int64(0)], 78)
+        self.assertEqual(result[np.int64(1)], 75)
+
+
+if __name__ == "__main__":
+    unittest.main()