Update DPatch

Signed-off-by: Beat Buesser <[email protected]>

Author: Beat Buesser

art/attacks/evasion/dpatch.py

@@ -78,7 +78,7 @@ def __init__(
         self.learning_rate = learning_rate
         self.max_iter = max_iter
         self.batch_size = batch_size
-        self._patch = np.ones(shape=patch_shape) * (self.estimator.clip_values[1] + self.estimator.clip_values[0]) / 2.0
+        self._patch = np.random.randint(self.estimator.clip_values[0], self.estimator.clip_values[1], size=patch_shape).astype(np.float32)
         self._check_params()
 
         self.target_label = []
@@ -87,7 +87,7 @@ def generate(
         self,
         x: np.ndarray,
         y: Optional[np.ndarray] = None,
-        target_label: Union[int, List[int], np.ndarray] = 0,
+        target_label: Optional[Union[int, List[int], np.ndarray]] = None,
         **kwargs
     ) -> np.ndarray:
         """
@@ -105,16 +105,17 @@ def generate(
             raise ValueError("The DPatch attack does not use target labels.")
         if x.ndim != 4:
             raise ValueError("The adversarial patch can only be applied to images.")
-        if isinstance(target_label, int):
-            self.target_label = [target_label] * x.shape[0]
-        elif isinstance(target_label, np.ndarray):
-            if not (target_label.shape == (x.shape[0], 1) or target_label.shape == (x.shape[0],)):
-                raise ValueError("The target_label has to be a 1-dimensional array.")
-            self.target_label = target_label.tolist()
-        else:
-            if not len(target_label) == x.shape[0] or not isinstance(target_label, list):
-                raise ValueError("The target_label as list of integers needs to of length number of images in `x`.")
-            self.target_label = target_label
+        if target_label is not None:
+            if isinstance(target_label, int):
+                self.target_label = [target_label] * x.shape[0]
+            elif isinstance(target_label, np.ndarray):
+                if not (target_label.shape == (x.shape[0], 1) or target_label.shape == (x.shape[0],)):
+                    raise ValueError("The target_label has to be a 1-dimensional array.")
+                self.target_label = target_label.tolist()
+            else:
+                if not len(target_label) == x.shape[0] or not isinstance(target_label, list):
+                    raise ValueError("The target_label as list of integers needs to of length number of images in `x`.")
+                self.target_label = target_label
 
         for i_step in trange(self.max_iter, desc="DPatch iteration"):
             if i_step == 0 or (i_step + 1) % 100 == 0:
@@ -125,19 +126,32 @@ def generate(
             )
             patch_target: List[Dict[str, np.ndarray]] = list()
 
-            for i_image in range(patched_images.shape[0]):
+            if self.target_label:
+
+                for i_image in range(patched_images.shape[0]):
+                    i_x_1 = transforms[i_image]["i_x_1"]
+                    i_x_2 = transforms[i_image]["i_x_2"]
+                    i_y_1 = transforms[i_image]["i_y_1"]
+                    i_y_2 = transforms[i_image]["i_y_2"]
+
+                    target_dict = dict()
+                    target_dict["boxes"] = np.asarray([[i_x_1, i_y_1, i_x_2, i_y_2]])
+                    target_dict["labels"] = np.asarray([self.target_label[i_image], ])
+                    target_dict["scores"] = np.asarray([1.0, ])
 
-                i_x_1 = transforms[i_image]["i_x_1"]
-                i_x_2 = transforms[i_image]["i_x_2"]
-                i_y_1 = transforms[i_image]["i_y_1"]
-                i_y_2 = transforms[i_image]["i_y_2"]
+                    patch_target.append(target_dict)
 
-                target_dict = dict()
-                target_dict["boxes"] = np.asarray([[i_x_1, i_y_1, i_x_2, i_y_2]])
-                target_dict["labels"] = np.asarray([self.target_label[i_image],])
-                target_dict["scores"] = np.asarray([1.0,])
+            else:
 
-                patch_target.append(target_dict)
+                predictions = self.estimator.predict(x=patched_images)
+
+                for i_image in range(patched_images.shape[0]):
+                    target_dict = dict()
+                    target_dict["boxes"] = predictions[i_image]["boxes"].detach().cpu().numpy()
+                    target_dict["labels"] = predictions[i_image]["labels"].detach().cpu().numpy()
+                    target_dict["scores"] = predictions[i_image]["scores"].detach().cpu().numpy()
+
+                    patch_target.append(target_dict)
 
             num_batches = math.ceil(x.shape[0] / self.batch_size)
             patch_gradients = np.zeros_like(self._patch)
@@ -162,9 +176,13 @@ def generate(
                     else:
                         patch_gradients_i = gradients[i_image, i_x_1:i_x_2, i_y_1:i_y_2, :]
 
-                    patch_gradients += patch_gradients_i
+                    patch_gradients = patch_gradients + patch_gradients_i
+
+            if self.target_label:
+                self._patch = self._patch - np.sign(patch_gradients) * self.learning_rate
+            else:
+                self._patch = self._patch + np.sign(patch_gradients) * self.learning_rate
 
-            self._patch -= patch_gradients * self.learning_rate
             self._patch = np.clip(
                 self._patch, a_min=self.estimator.clip_values[0], a_max=self.estimator.clip_values[1],
             )

art/estimators/object_detection/pytorch_faster_rcnn.py

@@ -19,7 +19,7 @@
 This module implements the task specific estimator for Faster R-CNN v3 in PyTorch.
 """
 import logging
-from typing import List, Optional, Tuple, Union, TYPE_CHECKING
+from typing import List, Dict, Optional, Tuple, Union, TYPE_CHECKING
 
 import numpy as np
 
@@ -29,6 +29,7 @@
 
 if TYPE_CHECKING:
     # pylint: disable=C0412
+    import torch
     import torchvision
 
     from art.utils import CLIP_VALUES_TYPE, PREPROCESSING_TYPE
@@ -134,7 +135,7 @@ def __init__(
         self._model.eval()
         self.attack_losses: Tuple[str, ...] = attack_losses
 
-    def loss_gradient(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray:
+    def loss_gradient(self, x: np.ndarray, y: List[Dict[str, np.ndarray]], **kwargs) -> np.ndarray:
         """
         Compute the gradient of the loss function w.r.t. `x`.
 
@@ -158,9 +159,9 @@ def loss_gradient(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray:
 
         if y is not None:
             for i, y_i in enumerate(y):
-                y[i]["boxes"] = torch.tensor(y_i["boxes"], dtype=torch.float).to(self._device)
-                y[i]["labels"] = torch.tensor(y_i["labels"], dtype=torch.int64).to(self._device)
-                y[i]["scores"] = torch.tensor(y_i["scores"]).to(self._device)
+                y[i]["boxes"] = torch.from_numpy(y_i["boxes"]).type(torch.float).to(self._device)
+                y[i]["labels"] = torch.from_numpy(y_i["labels"]).type(torch.int64).to(self._device)
+                y[i]["scores"] = torch.from_numpy(y_i["scores"]).to(self._device)
 
         transform = torchvision.transforms.Compose([torchvision.transforms.ToTensor()])
         image_tensor_list = list()
@@ -207,13 +208,13 @@ def loss_gradient(self, x: np.ndarray, y: np.ndarray, **kwargs) -> np.ndarray:
 
         return grads
 
-    def predict(self, x: np.ndarray, batch_size: int = 128, **kwargs) -> np.ndarray:
+    def predict(self, x: np.ndarray, batch_size: int = 128, **kwargs) -> List[Dict[str, "torch.Tensor"]]:
         """
         Perform prediction for a batch of inputs.
 
         :param x: Samples of shape (nb_samples, height, width, nb_channels).
         :param batch_size: Batch size.
-        :return: Predictions of format `List[Dict[Tensor]]`, one for each input image. The
+        :return: Predictions of format `List[Dict[str, Tensor]]`, one for each input image. The
                  fields of the Dict are as follows:
 
                  - boxes (FloatTensor[N, 4]): the predicted boxes in [x1, y1, x2, y2] format, with values \