Add test and fix weird keras behavior

Author: stefanradev93

bayesflow/diagnostics/metrics/classifier_two_sample_test.py

@@ -12,13 +12,13 @@ def classifier_two_sample_test(
     estimates: np.ndarray,
     targets: np.ndarray,
     metric: str = "accuracy",
-    patience: int = 10,
+    patience: int = 5,
     max_epochs: int = 1000,
-    batch_size: int = 64,
+    batch_size: int = 128,
     return_metric_only: bool = True,
     validation_split: float = 0.5,
     standardize: bool = True,
-    mlp_widths: Sequence = (256, 256),
+    mlp_widths: Sequence = (64, 64),
     **kwargs,
 ) -> float | Mapping[str, Any]:
     """
@@ -95,6 +95,11 @@ def classifier_two_sample_test(
     data = np.r_[estimates, targets]
     labels = np.r_[np.zeros((estimates.shape[0],)), np.ones((targets.shape[0],))]
 
+    # Important: needed, since keras does not shuffle before selecting validation split
+    shuffle_idx = np.random.permutation(data.shape[0])
+    data = data[shuffle_idx]
+    labels = labels[shuffle_idx]
+
     # Create and train classifier with optional stopping
     classifier = keras.Sequential(
         [MLP(widths=mlp_widths, **kwargs.get("mlp_kwargs", {})), keras.layers.Dense(1, activation="sigmoid")]

tests/test_diagnostics/conftest.py

@@ -10,12 +10,12 @@ def var_names():
 
 @pytest.fixture()
 def random_samples_a():
-    return np.random.normal(loc=0, scale=1, size=(1000, 8))
+    return np.random.normal(loc=0, scale=1, size=(5000, 8))
 
 
 @pytest.fixture()
 def random_samples_b():
-    return np.random.normal(loc=0, scale=3, size=(1000, 8))
+    return np.random.normal(loc=0, scale=3, size=(5000, 8))
 
 
 @pytest.fixture()

tests/test_diagnostics/test_diagnostics_metrics.py

@@ -52,10 +52,10 @@ def test_root_mean_squared_error(random_estimates, random_targets):
 
 def test_classifier_two_sample_test(random_samples_a, random_samples_b):
     metric = bf.diagnostics.metrics.classifier_two_sample_test(estimates=random_samples_a, targets=random_samples_a)
-    assert 0.6 > metric > 0.4
+    assert 0.55 > metric > 0.45
 
     metric = bf.diagnostics.metrics.classifier_two_sample_test(estimates=random_samples_a, targets=random_samples_b)
-    assert metric > 0.6
+    assert metric > 0.55
 
 
 def test_expected_calibration_error(pred_models, true_models, model_names):