Merge pull request #1218 from Trusted-AI/dev_1.7.1

Update to ART 1.7.1

Author: beat-buesser

.github/actions/deepspeech-v2/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh -l
+#!/bin/bash
 
 exit_code=0
 

.github/actions/deepspeech-v3/run.sh

@@ -1,4 +1,4 @@
-#!/bin/sh -l
+#!/bin/bash
 
 exit_code=0
 

art/__init__.py

@@ -12,7 +12,7 @@
 from art import preprocessing
 
 # Semantic Version
-__version__ = "1.7.0"
+__version__ = "1.7.1.dev0"
 
 # pylint: disable=C0103
 

art/attacks/evasion/imperceptible_asr/imperceptible_asr_pytorch.py

@@ -317,15 +317,12 @@ class only supports targeted attack.
         theta_batch = []
         original_max_psd_batch = []
 
-        for i in range(len(x)):
-            theta, original_max_psd = self._compute_masking_threshold(original_input[i])
+        for _, x_i in enumerate(x):
+            theta, original_max_psd = self._compute_masking_threshold(x_i)
             theta = theta.transpose(1, 0)
             theta_batch.append(theta)
             original_max_psd_batch.append(original_max_psd)
 
-        theta_batch = np.array(theta_batch)
-        original_max_psd_batch = np.array(original_max_psd_batch)
-
         # Reset delta with new result
         local_batch_shape = successful_adv_input_1st_stage.shape
         self.global_optimal_delta.data = torch.zeros(self.batch_size, self.global_max_length).type(torch.float64)
@@ -485,7 +482,7 @@ def _forward_1st_stage(
         return loss, local_delta, decoded_output, masked_adv_input, local_delta_rescale
 
     def _attack_2nd_stage(
-        self, x: np.ndarray, y: np.ndarray, theta_batch: np.ndarray, original_max_psd_batch: np.ndarray
+        self, x: np.ndarray, y: np.ndarray, theta_batch: List[np.ndarray], original_max_psd_batch: List[np.ndarray]
     ) -> "torch.Tensor":
         """
         The second stage of the attack.
@@ -544,6 +541,7 @@ class only supports targeted attack.
                 local_delta_rescale=local_delta_rescale,
                 theta_batch=theta_batch,
                 original_max_psd_batch=original_max_psd_batch,
+                real_lengths=real_lengths,
             )
 
             # Total loss
@@ -597,15 +595,17 @@ class only supports targeted attack.
     def _forward_2nd_stage(
         self,
         local_delta_rescale: "torch.Tensor",
-        theta_batch: np.ndarray,
-        original_max_psd_batch: np.ndarray,
+        theta_batch: List[np.ndarray],
+        original_max_psd_batch: List[np.ndarray],
+        real_lengths: np.ndarray,
     ) -> "torch.Tensor":
         """
         The forward pass of the second stage of the attack.
 
         :param local_delta_rescale: Local delta after rescaled.
         :param theta_batch: Original thresholds.
         :param original_max_psd_batch: Original maximum psd.
+        :param real_lengths: Real lengths of original sequences.
         :return: The loss tensor of the second stage of the attack.
         """
         import torch  # lgtm [py/repeated-import]
@@ -616,7 +616,7 @@ def _forward_2nd_stage(
 
         for i, _ in enumerate(theta_batch):
             psd_transform_delta = self._psd_transform(
-                delta=local_delta_rescale[i, :], original_max_psd=original_max_psd_batch[i]
+                delta=local_delta_rescale[i, : real_lengths[i]], original_max_psd=original_max_psd_batch[i]
             )
 
             loss = torch.mean(relu(psd_transform_delta - torch.tensor(theta_batch[i]).to(self.estimator.device)))

art/attacks/evasion/over_the_air_flickering/over_the_air_flickering_pytorch.py

@@ -271,17 +271,17 @@ def _get_loss_gradients(self, x: "torch.Tensor", y: "torch.Tensor", perturbation
 
             # calculate regularization terms
             # thickness - loss term
-            perturbation = perturbation + eps
-            norm_reg = torch.mean(perturbation ** 2) + 1e-12
-            perturbation_roll_right = torch.roll(perturbation, 1, dims=0)
-            perturbation_roll_left = torch.roll(perturbation, -1, dims=0)
+            perturbation_i = perturbation[[i]] + eps
+            norm_reg = torch.mean(perturbation_i ** 2) + 1e-12
+            perturbation_roll_right = torch.roll(perturbation_i, 1, dims=1)
+            perturbation_roll_left = torch.roll(perturbation_i, -1, dims=1)
 
             # 1st order diff - loss term
-            diff_norm_reg = torch.mean((perturbation - perturbation_roll_right) ** 2) + 1e-12
+            diff_norm_reg = torch.mean((perturbation_i - perturbation_roll_right) ** 2) + 1e-12
 
             # 2nd order diff - loss term
             laplacian_norm_reg = (
-                torch.mean((-2 * perturbation + perturbation_roll_right + perturbation_roll_left) ** 2) + 1e-12
+                torch.mean((-2 * perturbation_i + perturbation_roll_right + perturbation_roll_left) ** 2) + 1e-12
             )
 
             regularization_loss = self.beta_0 * (

art/attacks/evasion/pixel_threshold.py

@@ -44,6 +44,7 @@
 from scipy.optimize import OptimizeResult, minimize
 from tqdm.auto import tqdm
 
+from art.config import ART_NUMPY_DTYPE
 from art.attacks.attack import EvasionAttack
 from art.estimators.estimator import BaseEstimator, NeuralNetworkMixin
 from art.estimators.classification.classifier import ClassifierMixin
@@ -128,7 +129,7 @@ def _check_params(self) -> None:
 
     def rescale_input(self, x):
         """Rescale inputs"""
-        x = x.astype(np.float32) / 255.0
+        x = x.astype(ART_NUMPY_DTYPE) / 255.0
         x = (x * (self.estimator.clip_values[1] - self.estimator.clip_values[0])) + self.estimator.clip_values[0]
         return x
 
@@ -175,7 +176,7 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
             x = (x - self.estimator.clip_values[0]) / (self.estimator.clip_values[1] - self.estimator.clip_values[0])
             x = x * 255.0
 
-        x = x.astype(np.uint8)
+        x = x.astype(ART_NUMPY_DTYPE)
 
         adv_x_best = []
         self.adv_th = []

art/attacks/inference/membership_inference/black_box.py

@@ -275,7 +275,7 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
             self.attack_model.eval()  # type: ignore
             inferred: Optional[np.ndarray] = None
             test_set = self._get_attack_dataset(f_1=features, f_2=y)
-            test_loader = DataLoader(test_set, batch_size=self.batch_size, shuffle=True, num_workers=0)
+            test_loader = DataLoader(test_set, batch_size=self.batch_size, shuffle=False, num_workers=0)
             for input1, input2, _ in test_loader:
                 input1, input2 = to_cuda(input1), to_cuda(input2)
                 outputs = self.attack_model(input1, input2)  # type: ignore

art/defences/detector/poison/spectral_signature_defense.py

@@ -76,7 +76,7 @@ def __init__(
         self.batch_size = batch_size
         self.eps_multiplier = eps_multiplier
         self.expected_pp_poison = expected_pp_poison
-        self.y_train_sparse = np.argmax(y_train, axis=1)
+        self.y_train = y_train
         self.evaluator = GroundTruthEvaluator()
         self._check_params()
 
@@ -91,9 +91,9 @@ def evaluate_defence(self, is_clean: np.ndarray, **kwargs) -> str:
         """
         if is_clean is None or is_clean.size == 0:
             raise ValueError("is_clean was not provided while invoking evaluate_defence.")
-        is_clean_by_class = segment_by_class(is_clean, self.y_train_sparse, self.classifier.nb_classes)
+        is_clean_by_class = segment_by_class(is_clean, self.y_train, self.classifier.nb_classes)
         _, predicted_clean = self.detect_poison()
-        predicted_clean_by_class = segment_by_class(predicted_clean, self.y_train_sparse, self.classifier.nb_classes)
+        predicted_clean_by_class = segment_by_class(predicted_clean, self.y_train, self.classifier.nb_classes)
 
         _, conf_matrix_json = self.evaluator.analyze_correctness(predicted_clean_by_class, is_clean_by_class)
 
@@ -118,7 +118,7 @@ def detect_poison(self, **kwargs) -> Tuple[dict, List[int]]:
             self.x_train, layer=nb_layers - 1, batch_size=self.batch_size
         )
 
-        features_split = segment_by_class(features_x_poisoned, self.y_train_sparse, self.classifier.nb_classes)
+        features_split = segment_by_class(features_x_poisoned, self.y_train, self.classifier.nb_classes)
         score_by_class = []
         keep_by_class = []
 
@@ -134,11 +134,11 @@ def detect_poison(self, **kwargs) -> Tuple[dict, List[int]]:
                 keep_by_class.append([True])
 
         base_indices_by_class = segment_by_class(
-            np.arange(len(self.y_train_sparse)),
-            self.y_train_sparse,
+            np.arange(self.y_train.shape[0]),
+            self.y_train,
             self.classifier.nb_classes,
         )
-        is_clean_lst = [0] * len(self.y_train_sparse)
+        is_clean_lst = [0] * self.y_train.shape[0]
         report = {}
 
         for keep_booleans, all_scores, indices in zip(keep_by_class, score_by_class, base_indices_by_class):
@@ -171,5 +171,5 @@ def spectral_signature_scores(matrix_r: np.ndarray) -> np.ndarray:
         _, _, matrix_v = np.linalg.svd(matrix_m, full_matrices=False)
         eigs = matrix_v[:1]
         corrs = np.matmul(eigs, np.transpose(matrix_r))
-        score = np.expand_dims(np.linalg.norm(corrs, axis=1), axis=1)
+        score = np.expand_dims(np.linalg.norm(corrs, axis=0), axis=1)
         return score

art/defences/preprocessor/__init__.py

@@ -3,10 +3,11 @@
 """
 from art.defences.preprocessor.feature_squeezing import FeatureSqueezing
 from art.defences.preprocessor.gaussian_augmentation import GaussianAugmentation
-from art.defences.preprocessor.inverse_gan import InverseGAN, DefenseGAN
+from art.defences.preprocessor.inverse_gan import DefenseGAN, InverseGAN
 from art.defences.preprocessor.jpeg_compression import JpegCompression
 from art.defences.preprocessor.label_smoothing import LabelSmoothing
 from art.defences.preprocessor.mp3_compression import Mp3Compression
+from art.defences.preprocessor.mp3_compression_pytorch import Mp3CompressionPyTorch
 from art.defences.preprocessor.pixel_defend import PixelDefend
 from art.defences.preprocessor.preprocessor import Preprocessor
 from art.defences.preprocessor.resample import Resample
@@ -16,3 +17,4 @@
 from art.defences.preprocessor.thermometer_encoding import ThermometerEncoding
 from art.defences.preprocessor.variance_minimization import TotalVarMin
 from art.defences.preprocessor.video_compression import VideoCompression
+from art.defences.preprocessor.video_compression_pytorch import VideoCompressionPyTorch

art/defences/preprocessor/mp3_compression.py

@@ -72,7 +72,7 @@ def __call__(self, x: np.ndarray, y: Optional[np.ndarray] = None) -> Tuple[np.nd
         Apply MP3 compression to sample `x`.
 
         :param x: Sample to compress with shape `(batch_size, length, channel)` or an array of sample arrays with shape
-                  (length,) or (length, channel). `x` values are recommended to be of type `np.int16`.
+                  (length,) or (length, channel).
         :param y: Labels of the sample `x`. This function does not affect them in any way.
         :return: Compressed sample.
         """
@@ -84,11 +84,12 @@ def wav_to_mp3(x, sample_rate):
             from pydub import AudioSegment
             from scipy.io.wavfile import write
 
+            x_dtype = x.dtype
             normalized = bool(x.min() >= -1.0 and x.max() <= 1.0)
-            if x.dtype != np.int16 and not normalized:
+            if x_dtype != np.int16 and not normalized:
                 # input is not of type np.int16 and seems to be unnormalized. Therefore casting to np.int16.
                 x = x.astype(np.int16)
-            elif x.dtype != np.int16 and normalized:
+            elif x_dtype != np.int16 and normalized:
                 # x is not of type np.int16 and seems to be normalized. Therefore undoing normalization and
                 # casting to np.int16.
                 x = (x * 2 ** 15).astype(np.int16)
@@ -100,7 +101,19 @@ def wav_to_mp3(x, sample_rate):
             tmp_wav.close()
             tmp_mp3.close()
             x_mp3 = np.array(audio_segment.get_array_of_samples()).reshape((-1, audio_segment.channels))
-            return x_mp3
+
+            # WARNING: Sometimes we *still* need to manually resize x_mp3 to original length.
+            # This should not be the case, e.g. see https://github.com/jiaaro/pydub/issues/474
+            if x.shape[0] != x_mp3.shape[0]:
+                logger.warning(
+                    "Lengths original input and compressed output don't match. Truncating compressed result."
+                )
+                x_mp3 = x_mp3[: x.shape[0]]
+
+            if normalized:
+                # x was normalized. Therefore normalizing x_mp3.
+                x_mp3 = x_mp3 * 2 ** -15
+            return x_mp3.astype(x_dtype)
 
         if x.dtype != np.object and x.ndim != 3:
             raise ValueError("Mp3 compression can only be applied to temporal data across at least one channel.")