Merge branch 'dev_1.6.0' into development_issue_824

Author: beat-buesser

.github/workflows/ci.yml

@@ -31,24 +31,18 @@ jobs:
             tensorflow: 1.15.5
             tf_version: v1
             keras: 2.2.5
-          - name: TensorFlow 2.2.2 (Keras 2.3.1 Python 3.7)
-            framework: tensorflow
-            python: 3.7
-            tensorflow: 2.2.2
-            tf_version: v2
-            keras: 2.3.1
-          - name: TensorFlow 2.2.2v1 (Keras 2.3.1 Python 3.7)
-            framework: tensorflow2v1
-            python: 3.7
-            tensorflow: 2.2.2
-            tf_version: v2
-            keras: 2.3.1
           - name: TensorFlow 2.3.2 (Keras 2.4.3 Python 3.7)
             framework: tensorflow
             python: 3.7
             tensorflow: 2.3.2
             tf_version: v2
             keras: 2.4.3
+          - name: TensorFlow 2.4.1v1 (Keras 2.4.3 Python 3.8)
+            framework: tensorflow2v1
+            python: 3.8
+            tensorflow: 2.4.1
+            tf_version: v2
+            keras: 2.4.3
           - name: TensorFlow 2.4.1 (Keras 2.4.3 Python 3.8)
             framework: tensorflow
             python: 3.8
@@ -137,6 +131,8 @@ jobs:
         run: |
           pip install tensorflow==${{ matrix.tensorflow }}
           pip install lingvo==${{ matrix.lingvo }}
+          pip install tensorflow-addons==0.9.1
+          pip install model-pruning-google-research==0.0.3
           pip list
       - name: Run ${{ matrix.name }} Tests
         run: ./run_tests.sh ${{ matrix.framework }}

art/attacks/evasion/adversarial_patch/adversarial_patch_tensorflow.py

@@ -33,7 +33,7 @@
 from art.attacks.attack import EvasionAttack
 from art.estimators.estimator import BaseEstimator, NeuralNetworkMixin
 from art.estimators.classification.classifier import ClassifierMixin
-from art.utils import check_and_transform_label_format
+from art.utils import check_and_transform_label_format, is_probability
 
 if TYPE_CHECKING:
     import tensorflow as tf
@@ -111,6 +111,8 @@ def __init__(
         if self.estimator.channels_first:
             raise ValueError("Color channel needs to be in last dimension.")
 
+        self.use_logits = None
+
         self.i_h_patch = 0
         self.i_w_patch = 1
 
@@ -142,8 +144,8 @@ def __init__(
             constraint=lambda x: tf.clip_by_value(x, self.estimator.clip_values[0], self.estimator.clip_values[1]),
         )
 
-        self._train_op = tf.keras.optimizers.SGD(
-            learning_rate=self.learning_rate, momentum=0.0, nesterov=False, name="SGD"
+        self._train_op = tf.keras.optimizers.Adam(
+            learning_rate=self.learning_rate, beta_1=0.9, beta_2=0.999, epsilon=1e-07, amsgrad=False, name="Adam"
         )
 
     def _train_step(
@@ -170,7 +172,7 @@ def _train_step(
 
         return loss
 
-    def _probabilities(self, images: "tf.Tensor", mask: Optional["tf.Tensor"]) -> "tf.Tensor":
+    def _predictions(self, images: "tf.Tensor", mask: Optional["tf.Tensor"]) -> "tf.Tensor":
         import tensorflow as tf  # lgtm [py/repeated-import]
 
         patched_input = self._random_overlay(images, self._patch, mask=mask)
@@ -179,17 +181,17 @@ def _probabilities(self, images: "tf.Tensor", mask: Optional["tf.Tensor"]) -> "t
             patched_input, clip_value_min=self.estimator.clip_values[0], clip_value_max=self.estimator.clip_values[1],
         )
 
-        probabilities = self.estimator._predict_framework(patched_input)
+        predictions = self.estimator._predict_framework(patched_input)
 
-        return probabilities
+        return predictions
 
     def _loss(self, images: "tf.Tensor", target: "tf.Tensor", mask: Optional["tf.Tensor"]) -> "tf.Tensor":
         import tensorflow as tf  # lgtm [py/repeated-import]
 
-        probabilities = self._probabilities(images, mask)
+        predictions = self._predictions(images, mask)
 
         self._loss_per_example = tf.keras.losses.categorical_crossentropy(
-            y_true=target, y_pred=probabilities, from_logits=False, label_smoothing=0
+            y_true=target, y_pred=predictions, from_logits=self.use_logits, label_smoothing=0
         )
 
         loss = tf.reduce_mean(self._loss_per_example)
@@ -381,27 +383,21 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> T
         mask = kwargs.get("mask")
         if mask is not None:
             mask = mask.copy()
-        if mask is not None and (
-            (mask.dtype != np.bool)
-            or not (mask.shape[0] == 1 or mask.shape[0] == x.shape[0])
-            or not (
-                (mask.shape[1] == x.shape[1] and mask.shape[2] == x.shape[2])
-                or (mask.shape[1] == x.shape[2] and mask.shape[2] == x.shape[3])
-            )
-        ):
-            raise ValueError(
-                "The shape of `mask` has to be equal to the shape of a single samples (1, H, W) or the"
-                "shape of `x` (N, H, W) without their channel dimensions."
-            )
-
-        if mask is not None and mask.shape[0] == 1:
-            mask = np.repeat(mask, repeats=x.shape[0], axis=0)
+        mask = self._check_mask(mask=mask, x=x)
 
         if kwargs.get("reset_patch"):
             self.reset_patch(initial_patch_value=self._initial_value)
 
         y = check_and_transform_label_format(labels=y, nb_classes=self.estimator.nb_classes)
 
+        # check if logits or probabilities
+        y_pred = self.estimator.predict(x=x[[0]])
+
+        if is_probability(y_pred):
+            self.use_logits = False
+        else:
+            self.use_logits = True
+
         if mask is None:
             if shuffle:
                 dataset = (
@@ -444,6 +440,22 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> T
             self._get_circular_patch_mask(nb_samples=1).numpy()[0],
         )
 
+    def _check_mask(self, mask: np.ndarray, x: np.ndarray) -> np.ndarray:
+        if mask is not None and (
+            (mask.dtype != np.bool)
+            or not (mask.shape[0] == 1 or mask.shape[0] == x.shape[0])
+            or not (mask.shape[1] == x.shape[self.i_h + 1] and mask.shape[2] == x.shape[self.i_w + 1])
+        ):
+            raise ValueError(
+                "The shape of `mask` has to be equal to the shape of a single samples (1, H, W) or the"
+                "shape of `x` (N, H, W) without their channel dimensions."
+            )
+
+        if mask is not None and mask.shape[0] == 1:
+            mask = np.repeat(mask, repeats=x.shape[0], axis=0)
+
+        return mask
+
     def apply_patch(
         self,
         x: np.ndarray,
@@ -464,6 +476,7 @@ def apply_patch(
         """
         if mask is not None:
             mask = mask.copy()
+        mask = self._check_mask(mask=mask, x=x)
         patch = patch_external if patch_external is not None else self._patch
         return self._random_overlay(images=x, patch=patch, scale=scale, mask=mask).numpy()
 

art/attacks/evasion/dpatch.py

@@ -31,7 +31,6 @@
 from art.attacks.attack import EvasionAttack
 from art.estimators.estimator import BaseEstimator, LossGradientsMixin
 from art.estimators.object_detection.object_detector import ObjectDetectorMixin
-from art.utils import Deprecated, deprecated_keyword_arg
 from art import config
 
 if TYPE_CHECKING:
@@ -225,13 +224,11 @@ def generate(
         return self._patch
 
     @staticmethod
-    @deprecated_keyword_arg("channel_index", end_version="1.6.0", replaced_by="channels_first")
     def _augment_images_with_patch(
         x: np.ndarray,
         patch: np.ndarray,
         random_location: bool,
         channels_first: bool,
-        channel_index=Deprecated,
         mask: Optional[np.ndarray] = None,
     ) -> Tuple[np.ndarray, List[Dict[str, int]]]:
         """
@@ -242,21 +239,11 @@ def _augment_images_with_patch(
         :param random_location: If True apply patch at randomly shifted locations, otherwise place patch at origin
                                 (top-left corner).
         :param channels_first: Set channels first or last.
-        :param channel_index: Index of the color channel.
-        :type channel_index: `int`
         :param mask: An boolean array of shape equal to the shape of a single samples (1, H, W) or the shape of `x`
                      (N, H, W) without their channel dimensions. Any features for which the mask is True can be the
                      center location of the patch during sampling.
         :type mask: `np.ndarray`
         """
-        # Remove in 1.6.0
-        if channel_index == 3:
-            channels_first = False
-        elif channel_index == 1:
-            channels_first = True
-        elif channel_index is not Deprecated:
-            raise ValueError("Not a proper channel_index. Use channels_first.")
-
         transformations = list()
         x_copy = x.copy()
         patch_copy = patch.copy()

art/attacks/evasion/simba.py

@@ -29,7 +29,7 @@
 from scipy.fftpack import idct
 
 from art.attacks.attack import EvasionAttack
-from art.estimators.estimator import BaseEstimator
+from art.estimators.estimator import BaseEstimator, NeuralNetworkMixin
 from art.estimators.classification.classifier import ClassifierMixin
 from art.config import ART_NUMPY_DTYPE
 
@@ -51,7 +51,7 @@ class SimBA(EvasionAttack):
         "batch_size",
     ]
 
-    _estimator_requirements = (BaseEstimator, ClassifierMixin)
+    _estimator_requirements = (BaseEstimator, ClassifierMixin, NeuralNetworkMixin)
 
     def __init__(
         self,

art/attacks/evasion/square_attack.py

@@ -31,7 +31,7 @@
 
 from art.config import ART_NUMPY_DTYPE
 from art.attacks.attack import EvasionAttack
-from art.estimators.estimator import BaseEstimator
+from art.estimators.estimator import BaseEstimator, NeuralNetworkMixin
 from art.estimators.classification.classifier import ClassifierMixin
 from art.utils import check_and_transform_label_format, get_labels_np_array
 
@@ -53,7 +53,7 @@ class SquareAttack(EvasionAttack):
         "verbose",
     ]
 
-    _estimator_requirements = (BaseEstimator, ClassifierMixin)
+    _estimator_requirements = (BaseEstimator, ClassifierMixin, NeuralNetworkMixin)
 
     def __init__(
         self,
@@ -259,7 +259,7 @@ def _get_perturbation(h):
 
                     return delta
 
-                delta_init = np.zeros(x_robust.shape)
+                delta_init = np.zeros(x_robust.shape, dtype=ART_NUMPY_DTYPE)
 
                 height_start = 0
                 for _ in range(n_tiles):

art/attacks/inference/membership_inference/black_box.py

@@ -198,13 +198,7 @@ def fit(self, x: np.ndarray, y: np.ndarray, test_x: np.ndarray, test_y: np.ndarr
             import torch.nn as nn  # lgtm [py/repeated-import]
             import torch.optim as optim  # lgtm [py/repeated-import]
             from torch.utils.data import DataLoader  # lgtm [py/repeated-import]
-
-            use_cuda = torch.cuda.is_available()
-
-            def to_cuda(x):
-                if use_cuda:
-                    x = x.cuda()
-                return x
+            from art.utils import to_cuda
 
             loss_fn = nn.BCELoss()
             optimizer = optim.Adam(self.attack_model.parameters(), lr=self.learning_rate)
@@ -261,14 +255,18 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
         if self.default_model and self.attack_model_type == "nn":
             import torch  # lgtm [py/repeated-import]
             from torch.utils.data import DataLoader  # lgtm [py/repeated-import]
+            from art.utils import to_cuda, from_cuda
 
             self.attack_model.eval()
             inferred = None
             test_set = self._get_attack_dataset(f_1=features, f_2=y)
             test_loader = DataLoader(test_set, batch_size=self.batch_size, shuffle=True, num_workers=0)
             for input1, input2, _ in test_loader:
+                input1, input2 = to_cuda(input1), to_cuda(input2)
                 outputs = self.attack_model(input1, input2)
                 predicted = torch.round(outputs)
+                predicted = from_cuda(predicted)
+
                 if inferred is None:
                     inferred = predicted.detach().numpy()
                 else:

art/defences/detector/evasion/detector.py

@@ -28,7 +28,6 @@
 
 from art.estimators.estimator import BaseEstimator, NeuralNetworkMixin, LossGradientsMixin
 from art.estimators.classification.classifier import ClassifierMixin, ClassGradientsMixin
-from art.utils import deprecated
 
 if TYPE_CHECKING:
     from art.utils import CLIP_VALUES_TYPE
@@ -58,7 +57,6 @@ def __init__(self, detector: "ClassifierNeuralNetwork") -> None:
         super().__init__(
             model=None,
             clip_values=detector.clip_values,
-            channel_index=detector.channel_index,
             channels_first=detector.channels_first,
             preprocessing_defences=detector.preprocessing_defences,
             preprocessing=detector.preprocessing,
@@ -122,11 +120,6 @@ def input_shape(self) -> Tuple[int, ...]:
     def clip_values(self) -> Optional["CLIP_VALUES_TYPE"]:
         return self.detector.clip_values
 
-    @property  # type: ignore
-    @deprecated(end_version="1.6.0", replaced_by="channels_first")
-    def channel_index(self) -> Optional[int]:
-        return self.detector.channel_index
-
     @property
     def channels_first(self) -> Optional[bool]:
         """
@@ -189,7 +182,6 @@ def __init__(
         super().__init__(
             model=None,
             clip_values=detector.clip_values,
-            channel_index=detector.channel_index,
             channels_first=detector.channels_first,
             preprocessing_defences=detector.preprocessing_defences,
             preprocessing=detector.preprocessing,
@@ -270,11 +262,6 @@ def input_shape(self) -> Tuple[int, ...]:
     def clip_values(self) -> Optional["CLIP_VALUES_TYPE"]:
         return self.detector.clip_values
 
-    @property  # type: ignore
-    @deprecated(end_version="1.6.0", replaced_by="channels_first")
-    def channel_index(self) -> Optional[int]:
-        return self.detector.channel_index
-
     @property
     def channels_first(self) -> Optional[bool]:
         """

art/defences/detector/evasion/subsetscanning/detector.py

@@ -31,7 +31,6 @@
 
 from art.defences.detector.evasion.subsetscanning.scanner import Scanner
 from art.estimators.classification.classifier import ClassifierNeuralNetwork
-from art.utils import deprecated
 
 
 if TYPE_CHECKING:
@@ -68,7 +67,6 @@ def __init__(
         super().__init__(
             model=None,
             clip_values=classifier.clip_values,
-            channel_index=classifier.channel_index,
             channels_first=classifier.channels_first,
             preprocessing_defences=classifier.preprocessing_defences,
             preprocessing=classifier.preprocessing,
@@ -245,11 +243,6 @@ def input_shape(self) -> Tuple[int, ...]:
     def clip_values(self) -> Optional["CLIP_VALUES_TYPE"]:
         return self.detector.clip_values
 
-    @property  # type: ignore
-    @deprecated(end_version="1.6.0", replaced_by="channels_first")
-    def channel_index(self) -> Optional[int]:
-        return self.detector.channel_index
-
     @property
     def channels_first(self) -> bool:
         """