Merge pull request #556 from Trusted-AI/development_shadow_attack

Update test of ShadowAttack

Author: beat-buesser

art/attacks/evasion/shadow_attack.py

@@ -109,12 +109,16 @@ def __init__(
 
     def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
         """
-        Generate adversarial samples and return them in an array.
+        Generate adversarial samples and return them in an array. This requires a lot of memory, therefore it accepts
+        only a single samples as input, e.g. a batch of size 1.
 
-        :param x: An array with the original inputs.
-        :param y: An array with the target labels.
+        :param x: An array of a single original input sample.
+        :param y: An array of a single target label.
         :return: An array with the adversarial examples.
         """
+        if x.shape[0] > 1 or y.shape[0] > 1:
+            raise ValueError("This attack only accepts a single sample as input.")
+
         y = check_and_transform_label_format(y, self.estimator.nb_classes)
 
         if y is None:

run_tests.sh

@@ -9,6 +9,9 @@ export TF_CPP_MIN_LOG_LEVEL="3"
 pytest -q tests/attacks/evasion/ --mlFramework="tensorflow" --durations=0
 if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion tests"; fi
 
+pytest -q -s tests/attacks/evasion/test_shadow_attack.py --mlFramework="pytorch" --durations=0
+if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion/test_shadow_attack.py"; fi
+
 mlFrameworkList=("tensorflow" "scikitlearn")
 for mlFramework in "${mlFrameworkList[@]}"; do
   pytest -q tests/attacks/inference/ --mlFramework=$mlFramework --durations=0

tests/attacks/evasion/test_shadow_attack.py

@@ -19,6 +19,7 @@
 import pytest
 
 import numpy as np
+import tensorflow as tf
 
 from art.attacks.evasion import ShadowAttack
 from art.estimators.estimator import BaseEstimator, LossGradientsMixin
@@ -34,18 +35,15 @@ def fix_get_mnist_subset(get_mnist_dataset):
     (x_train_mnist, y_train_mnist), (x_test_mnist, y_test_mnist) = get_mnist_dataset
     n_train = 100
     n_test = 11
-    yield (x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test])
+    yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.only_with_platform("pytorch")
+@pytest.mark.skipif(tf.__version__[0] == "1", reason="Skip for TensorFlow 1.x")
+@pytest.mark.skipMlFramework("keras", "scikitlearn")
 def test_generate(fix_get_mnist_subset, get_image_classifier_list_for_attack):
 
     classifier_list = get_image_classifier_list_for_attack(ShadowAttack)
 
-    if classifier_list is None:
-        logging.warning("Couldn't perform  this test because no classifier is defined")
-        return
-
     for classifier in classifier_list:
         attack = ShadowAttack(
             estimator=classifier,
@@ -61,15 +59,13 @@ def test_generate(fix_get_mnist_subset, get_image_classifier_list_for_attack):
 
         (x_train_mnist, y_train_mnist, x_test_mnist, y_test_mnist) = fix_get_mnist_subset
 
-        if attack.framework == "pytorch":
-            x_train_mnist = x_train_mnist.transpose((0, 3, 1, 2))
-
         x_train_mnist_adv = attack.generate(x=x_train_mnist[0:1], y=y_train_mnist[0:1])
 
-        assert np.max(np.abs(x_train_mnist_adv - x_train_mnist[0:1])) == pytest.approx(0.34966960549354553, 0.06)
+        assert np.max(np.abs(x_train_mnist_adv - x_train_mnist[0:1])) == pytest.approx(0.34966960549354553, abs=0.06)
 
 
-@pytest.mark.only_with_platform("pytorch")
+@pytest.mark.skipif(tf.__version__[0] == "1", reason="Skip for TensorFlow 1.x")
+@pytest.mark.skipMlFramework("keras", "scikitlearn")
 def test_get_regularisation_loss_gradients(fix_get_mnist_subset, get_image_classifier_list_for_attack):
 
     classifier_list = get_image_classifier_list_for_attack(ShadowAttack)
@@ -90,9 +86,6 @@ def test_get_regularisation_loss_gradients(fix_get_mnist_subset, get_image_class
 
         (x_train_mnist, _, _, _) = fix_get_mnist_subset
 
-        if attack.framework == "pytorch":
-            x_train_mnist = x_train_mnist.transpose((0, 3, 1, 2))
-
         gradients = attack._get_regularisation_loss_gradients(x_train_mnist[0:1])
 
         gradients_expected = np.array(

tests/attacks/evasion/test_square_attack.py

@@ -52,7 +52,7 @@ def test_generate(fix_get_mnist_subset, get_image_classifier_list_for_attack):
 
         x_train_mnist_adv = attack.generate(x=x_train_mnist, y=y_train_mnist)
 
-        assert np.mean(np.abs(x_train_mnist_adv - x_train_mnist)) == pytest.approx(0.0535, abs=0.015)
+        assert np.mean(np.abs(x_train_mnist_adv - x_train_mnist)) == pytest.approx(0.03636, abs=0.015)
         assert np.max(np.abs(x_train_mnist_adv - x_train_mnist)) == pytest.approx(0.3, abs=0.05)