Merge pull request #964 from Trusted-AI/dev_1.5.3_new

Update to ART 1.5.3

Author: beat-buesser

art/__init__.py

@@ -11,7 +11,7 @@
 from art import wrappers
 
 # Semantic Version
-__version__ = "1.5.2"
+__version__ = "1.5.3-dev"
 
 # pylint: disable=C0103
 

art/attacks/evasion/adversarial_asr.py

@@ -47,6 +47,8 @@ class CarliniWagnerASR(ImperceptibleASR):
         "learning_rate",
         "max_iter",
         "batch_size",
+        "decrease_factor_eps",
+        "num_iter_decrease_eps",
     ]
 
     def __init__(
@@ -55,6 +57,8 @@ def __init__(
         eps: float = 2000.0,
         learning_rate: float = 100.0,
         max_iter: int = 1000,
+        decrease_factor_eps: float = 0.8,
+        num_iter_decrease_eps: int = 10,
         batch_size: int = 16,
     ):
         """
@@ -64,22 +68,31 @@ def __init__(
         :param eps: Initial max norm bound for adversarial perturbation.
         :param learning_rate: Learning rate of attack.
         :param max_iter: Number of iterations.
+        :param decrease_factor_eps: Decrease factor for epsilon (Paper default: 0.8).
+        :param num_iter_decrease_eps: Iterations after which to decrease epsilon if attack succeeds (Paper default: 10).
         :param batch_size: Batch size.
         """
         # pylint: disable=W0231
 
-        # re-implement init such that inherrited methods work
+        # re-implement init such that inherited methods work
         EvasionAttack.__init__(self, estimator=estimator)  # pylint: disable=W0233
         self.masker = None
         self.eps = eps
         self.learning_rate_1 = learning_rate
         self.max_iter_1 = max_iter
         self.max_iter_2 = 0
         self._targeted = True
+        self.decrease_factor_eps = decrease_factor_eps
+        self.num_iter_decrease_eps = num_iter_decrease_eps
         self.batch_size = batch_size
 
         # set remaining stage 2 params to some random values
         self.alpha = 0.1
         self.learning_rate_2 = 0.1
+        self.loss_theta_min = 0.0
+        self.increase_factor_alpha: float = 1.0
+        self.num_iter_increase_alpha: int = 1
+        self.decrease_factor_alpha: float = 1.0
+        self.num_iter_decrease_alpha: int = 1
 
         self._check_params()

art/attacks/evasion/imperceptible_asr/imperceptible_asr.py

@@ -1265,7 +1265,7 @@ def _kernel_grad(self, sv: np.ndarray, x_sample: np.ndarray) -> np.ndarray:
                 2
                 * self.model._gamma
                 * (-1)
-                * np.exp(-self.model._gamma * np.linalg.norm(x_sample - sv, ord=2))
+                * np.exp(-self.model._gamma * np.linalg.norm(x_sample - sv, ord=2) ** 2)
                 * (x_sample - sv)
             )
         elif self.model.kernel == "sigmoid":

art/attacks/evasion/imperceptible_asr/imperceptible_asr_pytorch.py

@@ -114,9 +114,7 @@ def _set_preprocessing(preprocessing: Union["PREPROCESSING_TYPE", "Preprocessor"
         from art.defences.preprocessor.preprocessor import Preprocessor
 
         if preprocessing is None:
-            from art.preprocessing.standardisation_mean_std.standardisation_mean_std import StandardisationMeanStd
-
-            return StandardisationMeanStd(mean=0.0, std=1.0)
+            return None
         elif isinstance(preprocessing, tuple):
             from art.preprocessing.standardisation_mean_std.standardisation_mean_std import StandardisationMeanStd
 

art/estimators/classification/scikitlearn.py

@@ -104,6 +104,7 @@ def __init__(
 
         # Super initialization
         super().__init__(
+            model=model,
             clip_values=clip_values,
             channels_first=channels_first,
             preprocessing_defences=preprocessing_defences,

art/estimators/estimator.py

@@ -19,15 +19,14 @@
 This module implements the abstract estimator `PyTorchEstimator` for PyTorch models.
 """
 import logging
-from typing import Any, Tuple
+from typing import TYPE_CHECKING, Any, List, Tuple
 
 import numpy as np
 
-from art.estimators.estimator import (
-    BaseEstimator,
-    LossGradientsMixin,
-    NeuralNetworkMixin,
-)
+from art.estimators.estimator import BaseEstimator, LossGradientsMixin, NeuralNetworkMixin
+
+if TYPE_CHECKING:
+    import torch
 
 logger = logging.getLogger(__name__)
 
@@ -53,7 +52,7 @@ def __init__(self, device_type: str = "gpu", **kwargs) -> None:
                be divided by the second one.
         :param device_type: Type of device on which the classifier is run, either `gpu` or `cpu`.
         """
-        import torch
+        import torch  # lgtm [py/repeated-import]
 
         preprocessing = kwargs.get("preprocessing")
         if isinstance(preprocessing, tuple):
@@ -151,7 +150,8 @@ def _apply_preprocessing(self, x, y, fit: bool = False, no_grad=True) -> Tuple[A
         :return: Tuple of `x` and `y` after applying the defences and standardisation.
         :rtype: Format as expected by the `model`
         """
-        import torch
+        import torch  # lgtm [py/repeated-import]
+
         from art.preprocessing.standardisation_mean_std.standardisation_mean_std import StandardisationMeanStd
         from art.preprocessing.standardisation_mean_std.standardisation_mean_std_pytorch import (
             StandardisationMeanStdPyTorch,
@@ -227,7 +227,8 @@ def _apply_preprocessing_gradient(self, x, gradients, fit=False):
         :return: Gradients after backward pass through preprocessing defences.
         :rtype: Format as expected by the `model`
         """
-        import torch
+        import torch  # lgtm [py/repeated-import]
+
         from art.preprocessing.standardisation_mean_std.standardisation_mean_std import StandardisationMeanStd
         from art.preprocessing.standardisation_mean_std.standardisation_mean_std_pytorch import (
             StandardisationMeanStdPyTorch,
@@ -281,3 +282,49 @@ def _apply_preprocessing_gradient(self, x, gradients, fit=False):
             raise NotImplementedError("The current combination of preprocessing types is not supported.")
 
         return gradients
+
+    def _set_layer(self, train: bool, layerinfo: List["torch.nn.modules.Module"]) -> None:
+        """
+        Set all layers that are an instance of `layerinfo` into training or evaluation mode.
+
+        :param train: False for evaluation mode.
+        :param layerinfo: List of module types.
+        """
+        import torch  # lgtm [py/repeated-import]
+
+        assert all([issubclass(l, torch.nn.modules.Module) for l in layerinfo])
+
+        def set_train(layer, layerinfo=layerinfo):
+            "Set layer into training mode if instance of `layerinfo`."
+            if isinstance(layer, tuple(layerinfo)):
+                layer.train()
+
+        def set_eval(layer, layerinfo=layerinfo):
+            "Set layer into evaluation mode if instance of `layerinfo`."
+            if isinstance(layer, tuple(layerinfo)):
+                layer.eval()
+
+        if train:
+            self._model.apply(set_train)
+        else:
+            self._model.apply(set_eval)
+
+    def set_dropout(self, train: bool) -> None:
+        """
+        Set all dropout layers into train or eval mode.
+
+        :param train: False for evaluation mode.
+        """
+        import torch  # lgtm [py/repeated-import]
+
+        self._set_layer(train=train, layerinfo=[torch.nn.modules.dropout._DropoutNd])
+
+    def set_batchnorm(self, train: bool) -> None:
+        """
+        Set all batch normalization layers into train or eval mode.
+
+        :param train: False for evaluation mode.
+        """
+        import torch  # lgtm [py/repeated-import]
+
+        self._set_layer(train=train, layerinfo=[torch.nn.modules.batchnorm._BatchNorm])

art/estimators/object_detection/tensorflow_faster_rcnn.py

@@ -22,23 +22,22 @@
 | Paper link: https://arxiv.org/abs/1512.02595
 """
 import logging
-from typing import List, Optional, Tuple, Union, TYPE_CHECKING
+from typing import TYPE_CHECKING, List, Optional, Tuple, Union
 
 import numpy as np
 
-from art.estimators.speech_recognition.speech_recognizer import SpeechRecognizerMixin
+from art import config
 from art.estimators.pytorch import PyTorchEstimator
+from art.estimators.speech_recognition.speech_recognizer import SpeechRecognizerMixin
 from art.utils import get_file
-from art import config
 
 if TYPE_CHECKING:
     import torch
-
     from deepspeech_pytorch.model import DeepSpeech
 
-    from art.utils import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
-    from art.defences.preprocessor.preprocessor import Preprocessor
     from art.defences.postprocessor.postprocessor import Postprocessor
+    from art.defences.preprocessor.preprocessor import Preprocessor
+    from art.utils import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
 
 logger = logging.getLogger(__name__)
 
@@ -124,7 +123,6 @@ def __init__(
                             if available otherwise run on CPU.
         """
         import torch  # lgtm [py/repeated-import]
-
         from deepspeech_pytorch.configs.inference_config import LMConfig
         from deepspeech_pytorch.enums import DecoderType
         from deepspeech_pytorch.utils import load_decoder, load_model
@@ -273,10 +271,9 @@ def predict(
         :param batch_size: Batch size.
         :param transcription_output: Indicate whether the function will produce probability or transcription as
                                      prediction output. If transcription_output is not available, then probability
-                                     output is returned.
-        :type transcription_output: `bool`
+                                     output is returned. Default: True
         :return: Predicted probability (if transcription_output False) or transcription (default, if
-                 transcription_output is True or None):
+                 transcription_output is True):
                  - Probability return is a tuple of (probs, sizes), where `probs` is the probability of characters of
                  shape (nb_samples, seq_length, nb_classes) and `sizes` is the real sequence length of shape
                  (nb_samples,).
@@ -344,9 +341,9 @@ def predict(
         result_outputs[batch_idx] = result_outputs_
 
         # Check if users want transcription outputs
-        transcription_output = kwargs.get("transcription_output")
+        transcription_output = kwargs.get("transcription_output", True)
 
-        if transcription_output is None or transcription_output is False:
+        if transcription_output is False:
             return result_outputs, result_output_sizes
 
         # Now users want transcription outputs
@@ -375,8 +372,10 @@ def loss_gradient(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray:
         x_ = np.empty(len(x), dtype=object)
         x_[:] = list(x)
 
-        # Put the model in the training mode
+        # Put the model in the training mode, otherwise CUDA can't backpropagate through the model.
+        # However, model uses batch norm layers which need to be frozen
         self._model.train()
+        self.set_batchnorm(train=False)
 
         # Apply preprocessing
         x_preprocessed, y_preprocessed = self._apply_preprocessing(x_, y, fit=False)
@@ -427,6 +426,8 @@ def loss_gradient(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray:
             results = np.array([i for i in results], dtype=x.dtype)
             assert results.shape == x.shape and results.dtype == x.dtype
 
+        # Unfreeze batch norm layers again
+        self.set_batchnorm(train=True)
         return results
 
     def fit(self, x: np.ndarray, y: np.ndarray, batch_size: int = 128, nb_epochs: int = 10, **kwargs) -> None:
@@ -582,7 +583,6 @@ def _transform_model_input(
         """
         import torch  # lgtm [py/repeated-import]
         import torchaudio
-
         from deepspeech_pytorch.loader.data_loader import _collate_fn
 
         # These parameters are needed for the transformation

art/estimators/pytorch.py

@@ -513,7 +513,9 @@ def _loss_gradient_per_batch(self, x: np.ndarray, y: np.ndarray) -> np.ndarray:
             gradient = gradient_padded[:length]
             gradients.append(gradient)
 
-        return np.array(gradients, dtype=object)
+        # for ragged input, use np.object dtype
+        dtype = np.float32 if x.ndim != 1 else np.object
+        return np.array(gradients, dtype=dtype)
 
     def _loss_gradient_per_sequence(self, x: np.ndarray, y: np.ndarray) -> np.ndarray:
         """
@@ -539,7 +541,9 @@ def _loss_gradient_per_sequence(self, x: np.ndarray, y: np.ndarray) -> np.ndarra
             gradient = self._sess.run(self._loss_gradient_op, feed_dict)
             gradients.append(np.squeeze(gradient))
 
-        return np.array(gradients, dtype=object)
+        # for ragged input, use np.object dtype
+        dtype = np.float32 if x.ndim != 1 else np.object
+        return np.array(gradients, dtype=dtype)
 
     def set_learning_phase(self, train: bool) -> None:
         raise NotImplementedError