Merge pull request #744 from Trusted-AI/audio_preprocessing

Implement impulse response filtering for sequential inputs

Author: beat-buesser

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_pytorch.py

@@ -32,6 +32,8 @@
 from tqdm import trange, tqdm
 
 from art.config import ART_NUMPY_DTYPE
+from art.estimators.estimator import BaseEstimator, LossGradientsMixin
+from art.estimators.classification.classifier import ClassifierMixin
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_numpy import (
     ProjectedGradientDescentCommon,
 )
@@ -40,7 +42,6 @@
 if TYPE_CHECKING:
     import torch
     from art.estimators.classification.pytorch import PyTorchClassifier
-    from art.estimators.object_detection.pytorch_faster_rcnn import PyTorchFasterRCNN
 
 logger = logging.getLogger(__name__)
 
@@ -54,9 +55,11 @@ class ProjectedGradientDescentPyTorch(ProjectedGradientDescentCommon):
     | Paper link: https://arxiv.org/abs/1706.06083
     """
 
+    _estimator_requirements = (BaseEstimator, LossGradientsMixin, ClassifierMixin)
+
     def __init__(
         self,
-        estimator: Union["PyTorchClassifier", "PyTorchFasterRCNN"],
+        estimator: Union["PyTorchClassifier"],
         norm: Union[int, float, str] = np.inf,
         eps: Union[int, float, np.ndarray] = 0.3,
         eps_step: Union[int, float, np.ndarray] = 0.1,

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_tensorflow_v2.py

@@ -32,6 +32,8 @@
 from tqdm import trange, tqdm
 
 from art.config import ART_NUMPY_DTYPE
+from art.estimators.estimator import BaseEstimator, LossGradientsMixin
+from art.estimators.classification.classifier import ClassifierMixin
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_numpy import (
     ProjectedGradientDescentCommon,
 )
@@ -53,6 +55,8 @@ class ProjectedGradientDescentTensorFlowV2(ProjectedGradientDescentCommon):
     | Paper link: https://arxiv.org/abs/1706.06083
     """
 
+    _estimator_requirements = (BaseEstimator, LossGradientsMixin, ClassifierMixin)
+
     def __init__(
         self,
         estimator: "TensorFlowV2Classifier",

art/defences/preprocessor/preprocessor.py

@@ -205,10 +205,16 @@ def get_gradient(x, grad):
 
             return x_grad
 
-        if x.shape == grad.shape:
+        if x.dtype == np.object:
+            x_grad_list = list()
+            for i, x_i in enumerate(x):
+                x_grad_list.append(get_gradient(x=x_i, grad=grad[i]))
+            x_grad = np.empty(x.shape[0], dtype=object)
+            x_grad[:] = list(x_grad_list)
+        elif x.shape == grad.shape:
             x_grad = get_gradient(x=x, grad=grad)
         else:
-            # Special case for lass gradients
+            # Special case for loss gradients
             x_grad = np.zeros_like(grad)
             for i in range(grad.shape[1]):
                 x_grad[:, i, ...] = get_gradient(x=x, grad=grad[:, i, ...])
@@ -268,35 +274,41 @@ def __call__(self, x: np.ndarray, y: Optional[np.ndarray] = None) -> Tuple[np.nd
         return result, y
 
     # Backward compatibility.
-    def _get_gradient(self, x: np.ndarray, grad: np.ndarray) -> np.ndarray:
-        """
-        Helper function for estimate_gradient
-        """
+    def estimate_gradient(self, x: np.ndarray, grad: np.ndarray) -> np.ndarray:
         import tensorflow as tf  # lgtm [py/repeated-import]
 
-        with tf.GradientTape() as tape:
-            x = tf.convert_to_tensor(x, dtype=config.ART_NUMPY_DTYPE)
-            tape.watch(x)
-            grad = tf.convert_to_tensor(grad, dtype=config.ART_NUMPY_DTYPE)
+        def get_gradient(x: np.ndarray, grad: np.ndarray) -> np.ndarray:
+            """
+            Helper function for estimate_gradient
+            """
 
-            x_prime = self.estimate_forward(x)
+            with tf.GradientTape() as tape:
+                x = tf.convert_to_tensor(x, dtype=config.ART_NUMPY_DTYPE)
+                tape.watch(x)
+                grad = tf.convert_to_tensor(grad, dtype=config.ART_NUMPY_DTYPE)
 
-        x_grad = tape.gradient(target=x_prime, sources=x, output_gradients=grad)
+                x_prime = self.estimate_forward(x)
 
-        x_grad = x_grad.numpy()
-        if x_grad.shape != x.shape:
-            raise ValueError("The input shape is {} while the gradient shape is {}".format(x.shape, x_grad.shape))
+            x_grad = tape.gradient(target=x_prime, sources=x, output_gradients=grad)
 
-        return x_grad
+            x_grad = x_grad.numpy()
+            if x_grad.shape != x.shape:
+                raise ValueError("The input shape is {} while the gradient shape is {}".format(x.shape, x_grad.shape))
 
-    # Backward compatibility.
-    def estimate_gradient(self, x: np.ndarray, grad: np.ndarray) -> np.ndarray:
-        if x.shape == grad.shape:
-            x_grad = self._get_gradient(x=x, grad=grad)
+            return x_grad
+
+        if x.dtype == np.object:
+            x_grad_list = list()
+            for i, x_i in enumerate(x):
+                x_grad_list.append(get_gradient(x=x_i, grad=grad[i]))
+            x_grad = np.empty(x.shape[0], dtype=object)
+            x_grad[:] = list(x_grad_list)
+        elif x.shape == grad.shape:
+            x_grad = get_gradient(x=x, grad=grad)
         else:
-            # Special case for lass gradients
+            # Special case for loss gradients
             x_grad = np.zeros_like(grad)
             for i in range(grad.shape[1]):
-                x_grad[:, i, ...] = self._get_gradient(x=x, grad=grad[:, i, ...])
+                x_grad[:, i, ...] = get_gradient(x=x, grad=grad[:, i, ...])
 
         return x_grad

art/defences/preprocessor/spatial_smoothing_pytorch.py

@@ -63,7 +63,7 @@ def __init__(
         """
         Create an instance of local spatial smoothing.
 
-        :window_size: Size of spatial smoothing window.
+        :param window_size: Size of spatial smoothing window.
         :param channels_first: Set channels first or last.
         :param clip_values: Tuple of the form `(min, max)` representing the minimum and maximum values allowed
                for features.

art/estimators/pytorch.py

@@ -150,7 +150,7 @@ def _apply_preprocessing(self, x, y, fit: bool = False, no_grad=True) -> Tuple[A
         else:
             input_is_tensor = False
 
-        if self.all_framework_preprocessing:
+        if self.all_framework_preprocessing and not (not input_is_tensor and x.dtype == np.object):
             if not input_is_tensor:
                 # Convert np arrays to torch tensors.
                 x = torch.tensor(x, device=self._device)
@@ -179,8 +179,9 @@ def chain_processes(x, y):
                 if y is not None:
                     y = y.cpu().numpy()
 
-        elif len(self.preprocessing) in [1, 2] and isinstance(
-            self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdPyTorch)
+        elif len(self.preprocessing) == 1 or (
+            len(self.preprocessing) == 2
+            and isinstance(self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdPyTorch))
         ):
             # Compatible with non-PyTorch defences if no chaining.
             for preprocess in self.preprocessing:
@@ -220,7 +221,12 @@ def _apply_preprocessing_gradient(self, x, gradients, fit=False):
         if not self.preprocessing:
             return gradients
 
-        if self.all_framework_preprocessing:
+        if isinstance(x, torch.Tensor):
+            input_is_tensor = True
+        else:
+            input_is_tensor = False
+
+        if self.all_framework_preprocessing and not (not input_is_tensor and x.dtype == np.object):
             # Convert np arrays to torch tensors.
             x = torch.tensor(x, device=self._device, requires_grad=True)
             gradients = torch.tensor(gradients, device=self._device)
@@ -243,8 +249,9 @@ def _apply_preprocessing_gradient(self, x, gradients, fit=False):
                     "The input shape is {} while the gradient shape is {}".format(x.shape, gradients.shape)
                 )
 
-        elif len(self.preprocessing) in [1, 2] and isinstance(
-            self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdPyTorch)
+        elif len(self.preprocessing) == 1 or (
+                len(self.preprocessing) == 2
+                and isinstance(self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdPyTorch))
         ):
             # Compatible with non-PyTorch defences if no chaining.
             defence = self.preprocessing[0]

art/estimators/speech_recognition/pytorch_deep_speech.py

@@ -36,7 +36,7 @@
 
     from deepspeech_pytorch.model import DeepSpeech
 
-    from art.config import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
+    from art.utils import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
     from art.defences.preprocessor.preprocessor import Preprocessor
     from art.defences.postprocessor.postprocessor import Postprocessor
 
@@ -131,7 +131,9 @@ def __init__(
 
         # Super initialization
         super().__init__(
+            model=None,
             clip_values=clip_values,
+            channels_first=None,
             preprocessing_defences=preprocessing_defences,
             postprocessing_defences=postprocessing_defences,
             preprocessing=preprocessing,

art/estimators/speech_recognition/tensorflow_lingvo.py

@@ -136,6 +136,7 @@ def __init__(
 
         # Super initialization
         super().__init__(
+            model=None,
             clip_values=clip_values,
             channels_first=channels_first,
             preprocessing_defences=preprocessing_defences,

art/estimators/tensorflow.py

@@ -194,7 +194,7 @@ def _apply_preprocessing(self, x, y, fit: bool = False) -> Tuple[Any, Any]:
         else:
             input_is_tensor = False
 
-        if self.all_framework_preprocessing:
+        if self.all_framework_preprocessing and not (not input_is_tensor and x.dtype == np.object):
             # Convert np arrays to torch tensors.
             if not input_is_tensor:
                 x = tf.convert_to_tensor(x)
@@ -215,8 +215,9 @@ def _apply_preprocessing(self, x, y, fit: bool = False) -> Tuple[Any, Any]:
                 if y is not None:
                     y = y.numpy()
 
-        elif len(self.preprocessing) in [1, 2] and isinstance(
-            self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdTensorFlowV2)
+        elif len(self.preprocessing) == 1 or (
+            len(self.preprocessing) == 2
+            and isinstance(self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdTensorFlowV2))
         ):
             # Compatible with non-TensorFlow defences if no chaining.
             for preprocess in self.preprocessing:
@@ -256,7 +257,12 @@ def _apply_preprocessing_gradient(self, x, gradients, fit=False):
         if not self.preprocessing:
             return gradients
 
-        if self.all_framework_preprocessing:
+        if isinstance(x, tf.Tensor):
+            input_is_tensor = True
+        else:
+            input_is_tensor = False
+
+        if self.all_framework_preprocessing and not (not input_is_tensor and x.dtype == np.object):
             with tf.GradientTape() as tape:
                 # Convert np arrays to TensorFlow tensors.
                 x = tf.convert_to_tensor(x, dtype=config.ART_NUMPY_DTYPE)
@@ -281,8 +287,9 @@ def _apply_preprocessing_gradient(self, x, gradients, fit=False):
                     "The input shape is {} while the gradient shape is {}".format(x.shape, gradients.shape)
                 )
 
-        elif len(self.preprocessing) in [1, 2] and isinstance(
-            self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdTensorFlowV2)
+        elif len(self.preprocessing) == 1 or (
+                len(self.preprocessing) == 2
+                and isinstance(self.preprocessing[-1], (StandardisationMeanStd, StandardisationMeanStdTensorFlowV2))
         ):
             # Compatible with non-TensorFlow defences if no chaining.
             defence = self.preprocessing[0]

art/preprocessing/__init__.py

@@ -6,3 +6,5 @@
 from art.preprocessing.standardisation_mean_std.standardisation_mean_std_tensorflow import (
     StandardisationMeanStdTensorFlowV2,
 )
+from art.preprocessing.l_filter.l_filter import LFilter
+from art.preprocessing.l_filter.l_filter_pytorch import LFilterPyTorch