Update Feature Adversaries tests

Signed-off-by: Beat Buesser <[email protected]>

Author: Beat Buesser

art/attacks/evasion/feature_adversaries/feature_adversaries_pytorch.py

@@ -116,7 +116,7 @@ def _generate_batch(self, x: "torch.Tensor", y: "torch.Tensor") -> "torch.Tensor
         import torch  # lgtm [py/repeated-import]
 
         def loss_fn(source_orig, source_adv, guide):
-            representation_loss = torch.tensor([0.0]).to(self.estimator.device)
+            representation_loss = torch.zeros(size=(source_orig.shape[0],)).to(self.estimator.device)
             for layer_i in self.layer:
                 adv_representation = self.estimator.get_activations(source_adv, layer_i, self.batch_size, True)
                 guide_representation = self.estimator.get_activations(guide, layer_i, self.batch_size, True)

art/attacks/evasion/feature_adversaries/feature_adversaries_tensorflow.py

@@ -116,7 +116,7 @@ def _generate_batch(self, x: "tf.Tensor", y: "tf.Tensor") -> "tf.Tensor":
         import tensorflow as tf  # lgtm [py/repeated-import]
 
         def loss_fn(source_orig, source_adv, guide):
-            representation_loss = tf.constant([0.0], shape=(1,), dtype=tf.float32)
+            representation_loss = tf.zeros(shape=(source_orig.shape[0],), dtype=tf.float32)
             for layer_i in self.layer:
                 adv_representation = self.estimator.get_activations(source_adv, layer_i, self.batch_size, True)
                 guide_representation = self.estimator.get_activations(guide, layer_i, self.batch_size, True)

tests/attacks/evasion/feature_adversaries.py/test_feature_adversaries.py renamed to tests/attacks/evasion/feature_adversaries/test_feature_adversaries.py

@@ -42,13 +42,14 @@ def test_images_pgd(art_warning, fix_get_mnist_subset, image_dl_estimator_for_at
         (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
 
         classifier = image_dl_estimator_for_attack(FeatureAdversariesPyTorch, functional=True)
+        print("classifier", classifier)
 
         attack = FeatureAdversariesPyTorch(
-            classifier, delta=1.0, layer=1, batch_size=32, step_size=0.05, max_iter=1, random_start=False
+            classifier, delta=1.0, layer=1, batch_size=32, step_size=0.05, max_iter=2, random_start=False
         )
         x_train_mnist_adv = attack.generate(x=x_train_mnist[0:3], y=x_test_mnist[0:3])
         assert np.mean(x_train_mnist[0:3]) == pytest.approx(0.13015705, 0.01)
-        assert np.mean(x_train_mnist_adv) != 0
+        assert np.mean(np.abs(x_train_mnist_adv - x_train_mnist[0:3])) != 0.0
     except ARTTestException as e:
         art_warning(e)
 
@@ -67,14 +68,16 @@ def test_images_unconstrained_adam(art_warning, fix_get_mnist_subset, image_dl_e
         )
         x_train_mnist_adv = attack.generate(x=x_train_mnist[0:3], y=x_test_mnist[0:3])
         assert np.mean(x_train_mnist[0:3]) == pytest.approx(0.13015705, 0.01)
-        assert np.mean(x_train_mnist_adv) != 0
+        assert np.mean(np.abs(x_train_mnist_adv - x_train_mnist[0:3])) != 0.0
     except ARTTestException as e:
         art_warning(e)
 
 
-@pytest.mark.skip_framework("tensorflow", "keras", "kerastf", "mxnet", "non_dl_frameworks")
+@pytest.mark.framework_agnostic
 def test_classifier_type_check_fail(art_warning):
     try:
-        backend_test_classifier_type_check_fail(FeatureAdversariesPyTorch, [BaseEstimator, NeuralNetworkMixin])
+        backend_test_classifier_type_check_fail(
+            FeatureAdversariesPyTorch, [BaseEstimator, NeuralNetworkMixin], delta=1.0
+        )
     except ARTTestException as e:
         art_warning(e)

tests/attacks/evasion/feature_adversaries.py/test_feature_adversaries_pytorch.py renamed to tests/attacks/evasion/feature_adversaries/test_feature_adversaries_pytorch.py

@@ -44,11 +44,11 @@ def test_images_pgd(art_warning, fix_get_mnist_subset, image_dl_estimator_for_at
         classifier = image_dl_estimator_for_attack(FeatureAdversariesTensorFlowV2)
 
         attack = FeatureAdversariesTensorFlowV2(
-            classifier, delta=1.0, layer=1, batch_size=32, step_size=0.05, max_iter=1, random_start=False
+            classifier, delta=1.0, layer=1, batch_size=32, step_size=0.05, max_iter=2, random_start=False
         )
         x_train_mnist_adv = attack.generate(x=x_train_mnist[0:3], y=x_test_mnist[0:3])
         assert np.mean(x_train_mnist[0:3]) == pytest.approx(0.13015705, 0.01)
-        assert np.mean(x_train_mnist_adv) != 0
+        assert np.mean(np.abs(x_train_mnist_adv - x_train_mnist[0:3])) != 0.0
     except ARTTestException as e:
         art_warning(e)
 
@@ -67,14 +67,16 @@ def test_images_unconstrained_adam(art_warning, fix_get_mnist_subset, image_dl_e
         )
         x_train_mnist_adv = attack.generate(x=x_train_mnist[0:3], y=x_test_mnist[0:3])
         assert np.mean(x_train_mnist[0:3]) == pytest.approx(0.13015705, 0.01)
-        assert np.mean(x_train_mnist_adv) != 0
+        assert np.mean(np.abs(x_train_mnist_adv - x_train_mnist[0:3])) != 0.0
     except ARTTestException as e:
         art_warning(e)
 
 
-@pytest.mark.skip_framework("tensorflow1", "keras", "kerastf", "mxnet", "non_dl_frameworks", "pytorch")
+@pytest.mark.framework_agnostic
 def test_classifier_type_check_fail(art_warning):
     try:
-        backend_test_classifier_type_check_fail(FeatureAdversariesTensorFlowV2, [BaseEstimator, NeuralNetworkMixin])
+        backend_test_classifier_type_check_fail(
+            FeatureAdversariesTensorFlowV2, [BaseEstimator, NeuralNetworkMixin], delta=1.0
+        )
     except ARTTestException as e:
         art_warning(e)