adding label check to trades adversarial trainer

Signed-off-by: GiulioZizzo <[email protected]>

Author: GiulioZizzo

art/defences/trainer/adversarial_trainer_trades_pytorch.py

@@ -126,8 +126,13 @@ def fit(
             # compute accuracy
             if validation_data is not None:
                 (x_test, y_test) = validation_data
+
                 output = np.argmax(self.predict(x_test), axis=1)
-                nb_correct_pred = np.sum(output == np.argmax(y_test, axis=1))
+                if y_test.ndim > 1:
+                    nb_correct_pred = np.sum(output == np.argmax(y_test, axis=1))
+                else:
+                    nb_correct_pred = np.sum(output == y_test)
+
                 logger.info(
                     "epoch: %s time(s): %.1f loss: %.4f acc(tr): %.4f acc(val): %.4f",
                     i_epoch,
@@ -240,7 +245,7 @@ def _batch_process(self, x_batch: np.ndarray, y_batch: np.ndarray) -> Tuple[floa
         )
 
         # Check label shape
-        if self._classifier._reduce_labels:  # pylint: disable=W0212
+        if self._classifier._reduce_labels and y_preprocessed.ndim > 1:  # pylint: disable=W0212
             y_preprocessed = np.argmax(y_preprocessed, axis=1)
 
         i_batch = torch.from_numpy(x_preprocessed).to(self._classifier._device)  # pylint: disable=W0212

tests/defences/trainer/test_adversarial_trainer_trades_pytorch.py

@@ -63,22 +63,38 @@ def fix_get_mnist_subset(get_mnist_dataset):
     yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.skip_framework("tensorflow", "keras", "scikitlearn", "mxnet", "kerastf")
-def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix_get_mnist_subset):
+@pytest.mark.only_with_platform("pytorch")
+@pytest.mark.parametrize("label_format", ["one_hot", "numerical"])
+def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix_get_mnist_subset, label_format):
     (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
     x_test_mnist_original = x_test_mnist.copy()
 
+    if label_format == "one_hot":
+        assert y_train_mnist.shape[-1] == 10
+        assert y_test_mnist.shape[-1] == 10
+    if label_format == "numerical":
+        y_test_mnist = np.argmax(y_test_mnist, axis=1)
+        y_train_mnist = np.argmax(y_train_mnist, axis=1)
+
     trainer = get_adv_trainer()
     if trainer is None:
         logging.warning("Couldn't perform  this test because no trainer is defined for this framework configuration")
         return
 
     predictions = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+
+    if label_format == "one_hot":
+        accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy = np.sum(predictions == y_test_mnist) / x_test_mnist.shape[0]
 
     trainer.fit(x_train_mnist, y_train_mnist, nb_epochs=20)
     predictions_new = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+
+    if label_format == "one_hot":
+        accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy_new = np.sum(predictions_new == y_test_mnist) / x_test_mnist.shape[0]
 
     np.testing.assert_array_almost_equal(
         float(np.mean(x_test_mnist_original - x_test_mnist)),
@@ -92,13 +108,21 @@ def test_adversarial_trainer_trades_pytorch_fit_and_predict(get_adv_trainer, fix
     trainer.fit(x_train_mnist, y_train_mnist, nb_epochs=20, validation_data=(x_train_mnist, y_train_mnist))
 
 
-@pytest.mark.skip_framework("tensorflow", "keras", "scikitlearn", "mxnet", "kerastf")
+@pytest.mark.only_with_platform("pytorch")
+@pytest.mark.parametrize("label_format", ["one_hot", "numerical"])
 def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict(
-    get_adv_trainer, fix_get_mnist_subset, image_data_generator
+    get_adv_trainer, fix_get_mnist_subset, image_data_generator, label_format
 ):
     (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
     x_test_mnist_original = x_test_mnist.copy()
 
+    if label_format == "one_hot":
+        assert y_train_mnist.shape[-1] == 10
+        assert y_test_mnist.shape[-1] == 10
+    if label_format == "numerical":
+        y_test_mnist = np.argmax(y_test_mnist, axis=1)
+        y_train_mnist = np.argmax(y_train_mnist, axis=1)
+
     generator = image_data_generator()
 
     trainer = get_adv_trainer()
@@ -107,11 +131,18 @@ def test_adversarial_trainer_trades_pytorch_fit_generator_and_predict(
         return
 
     predictions = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    if label_format == "one_hot":
+        accuracy = np.sum(predictions == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy = np.sum(predictions == y_test_mnist) / x_test_mnist.shape[0]
 
     trainer.fit_generator(generator=generator, nb_epochs=20)
     predictions_new = np.argmax(trainer.predict(x_test_mnist), axis=1)
-    accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+
+    if label_format == "one_hot":
+        accuracy_new = np.sum(predictions_new == np.argmax(y_test_mnist, axis=1)) / x_test_mnist.shape[0]
+    else:
+        accuracy_new = np.sum(predictions_new == y_test_mnist) / x_test_mnist.shape[0]
 
     np.testing.assert_array_almost_equal(
         float(np.mean(x_test_mnist_original - x_test_mnist)),