Support rectangular images

Signed-off-by: Beat Buesser <[email protected]>

Author: Beat Buesser

art/attacks/evasion/adversarial_patch/adversarial_patch_numpy.py

@@ -24,6 +24,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
+import math
 from typing import Optional, Union
 
 import random
@@ -102,6 +103,9 @@ def __init__(
         self.clip_patch = clip_patch
         self._check_params()
 
+        if len(self.estimator.input_shape) not in [3]:
+            raise ValueError("Wrong input_shape in estimator detected. AdversarialPatch is expecting images as input.")
+
         self.image_shape = self.estimator.input_shape
 
         if self.estimator.channels_first:
@@ -148,7 +152,7 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         for _ in trange(self.max_iter, desc="Adversarial Patch Numpy"):
             patched_images, patch_mask_transformed, transforms = self._augment_images_with_random_patch(x, self.patch)
 
-            num_batches = int(x.shape[0] / self.batch_size)
+            num_batches = int(math.ceil(x.shape[0] / self.batch_size))
             patch_gradients = np.zeros_like(self.patch)
 
             for i_batch in range(num_batches):
@@ -159,7 +163,7 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
                     patched_images[i_batch_start:i_batch_end], y_target[i_batch_start:i_batch_end],
                 )
 
-                for i_image in range(self.batch_size):
+                for i_image in range(gradients.shape[0]):
                     patch_gradients_i = self._reverse_transformation(
                         gradients[i_image, :, :, :], patch_mask_transformed[i_image, :, :, :], transforms[i_image],
                     )
@@ -230,23 +234,24 @@ def _get_circular_patch_mask(self, sharpness: int = 40) -> np.ndarray:
 
         mask = 1 - np.clip(z_grid, -1, 1)
 
+        channel_index = 1 if self.estimator.channels_first else 3
+        axis = channel_index - 1
+        mask = np.expand_dims(mask, axis=axis)
+        mask = np.broadcast_to(mask, self.patch_shape).astype(np.float32)
+
         pad_h_before = int((self.image_shape[self.i_h] - mask.shape[self.i_h]) / 2)
         pad_h_after = int(self.image_shape[self.i_h] - pad_h_before - mask.shape[self.i_h])
 
         pad_w_before = int((self.image_shape[self.i_w] - mask.shape[self.i_w]) / 2)
         pad_w_after = int(self.image_shape[self.i_w] - pad_w_before - mask.shape[self.i_w])
 
-        mask = np.pad(
-            mask,
-            pad_width=((pad_h_before, pad_h_after), (pad_w_before, pad_w_after)),
-            mode="constant",
-            constant_values=(0, 0),
-        )
+        if self.estimator.channels_first:
+            pad_width = ((0, 0), (pad_h_before, pad_h_after), (pad_w_before, pad_w_after))
+        else:
+            pad_width = ((pad_h_before, pad_h_after), (pad_w_before, pad_w_after), (0, 0))
+
+        mask = np.pad(mask, pad_width=pad_width, mode="constant", constant_values=(0, 0),)
 
-        channel_index = 1 if self.estimator.channels_first else 3
-        axis = channel_index - 1
-        mask = np.expand_dims(mask, axis=axis)
-        mask = np.broadcast_to(mask, self.patch_shape).astype(np.float32)
         return mask
 
     def _augment_images_with_random_patch(self, images, patch, scale=None):
@@ -299,20 +304,22 @@ def _scale(self, x, scale):
             left = (width - scale_w) // 2
 
             x_out = np.zeros_like(x)
-            x_out[top : top + scale_h, left : left + scale_w] = zoom(x, zoom=zooms, order=1)
 
             if self.estimator.channels_first:
                 x_out[:, top : top + scale_h, left : left + scale_w] = zoom(x, zoom=zooms, order=1)
             else:
                 x_out[top : top + scale_h, left : left + scale_w, :] = zoom(x, zoom=zooms, order=1)
 
         elif scale > 1.0:
-            scale_h = int(np.round(height / scale))
-            scale_w = int(np.round(width / scale))
+            scale_h = int(np.round(height / scale)) + 1
+            scale_w = int(np.round(width / scale)) + 1
             top = (height - scale_h) // 2
             left = (width - scale_w) // 2
 
-            x_out = zoom(x[top : top + scale_h, left : left + scale_w], zoom=zooms, order=1)
+            if self.estimator.channels_first:
+                x_out = zoom(x[:, top : top + scale_h, left : left + scale_w], zoom=zooms, order=1)
+            else:
+                x_out = zoom(x[top : top + scale_h, left : left + scale_w, :], zoom=zooms, order=1)
 
             cut_top = (x_out.shape[self.i_h] - height) // 2
             cut_left = (x_out.shape[self.i_w] - width) // 2
@@ -325,16 +332,16 @@ def _scale(self, x, scale):
         else:
             x_out = x
 
+        assert x.shape == x_out.shape
+
         return x_out
 
     def _shift(self, x, shift_h, shift_w):
         if self.estimator.channels_first:
             shift_hw = (0, shift_h, shift_w)
         else:
             shift_hw = (shift_h, shift_w, 0)
-
-        x = shift(x, shift=shift_hw, order=1)
-        return x, shift_h, shift_w
+        return shift(x, shift=shift_hw, order=1)
 
     def _random_transformation(self, patch, scale):
         patch_mask = self._get_circular_patch_mask()
@@ -359,12 +366,13 @@ def _random_transformation(self, patch, scale):
         if shift_max_h > 0 and shift_max_w > 0:
             shift_h = random.uniform(-shift_max_h, shift_max_h)
             shift_w = random.uniform(-shift_max_w, shift_max_w)
-            patch, _, _ = self._shift(patch, shift_h, shift_w)
-            patch_mask, shift_1, shift_2 = self._shift(patch_mask, shift_h, shift_w)
-            transformation["shift_1"] = shift_1
-            transformation["shift_2"] = shift_2
+            patch = self._shift(patch, shift_h, shift_w)
+            patch_mask = self._shift(patch_mask, shift_h, shift_w)
+            transformation["shift_h"] = shift_h
+            transformation["shift_w"] = shift_w
         else:
-            transformation["shift"] = (0, 0, 0)
+            transformation["shift_h"] = 0
+            transformation["shift_w"] = 0
 
         return patch, patch_mask, transformation
 
@@ -374,7 +382,7 @@ def _reverse_transformation(self, gradients: np.ndarray, patch_mask_transformed,
         # shift
         shift_h = transformation["shift_h"]
         shift_w = transformation["shift_w"]
-        gradients, _, _ = self._shift(gradients, -shift_h, -shift_w)
+        gradients = self._shift(gradients, -shift_h, -shift_w)
 
         # scale
         scale = transformation["scale"]
@@ -383,4 +391,5 @@ def _reverse_transformation(self, gradients: np.ndarray, patch_mask_transformed,
         # rotate
         angle = transformation["rotate"]
         gradients = self._rotate(gradients, -angle)
+
         return gradients

art/attacks/evasion/adversarial_patch/adversarial_patch_tensorflow.py

@@ -105,15 +105,20 @@ def __init__(
         self.image_shape = classifier.input_shape
         self._check_params()
 
-        if self.image_shape[2] not in [1, 3]:
-            raise ValueError("Color channel need to be in last dimension.")
+        if self.estimator.channels_first:
+            raise ValueError("Color channel needs to be in last dimension.")
+
+        self.nb_dims = len(self.image_shape)
+        if self.nb_dims == 3:
+            self.i_h = 0
+            self.i_w = 1
+        elif self.nb_dims == 4:
+            self.i_h = 1
+            self.i_w = 2
 
         if self.patch_shape is None:
             self.patch_shape = self.estimator.input_shape
 
-        if self.patch_shape[2] not in [1, 3]:
-            raise ValueError("Color channel need to be in last dimension.")
-
         if self.patch_shape[0] != self.patch_shape[1]:
             raise ValueError("Patch height and width need to be the same.")
 
@@ -186,13 +191,13 @@ def _loss(self, images: "tf.Tensor", target: "tf.Tensor") -> "tf.Tensor":
 
         return loss
 
-    def _get_circular_patch_mask(self, nb_images: int, sharpness: int = 40) -> "tf.Tensor":
+    def _get_circular_patch_mask(self, nb_samples: int, sharpness: int = 40) -> "tf.Tensor":
         """
         Return a circular patch mask.
         """
         import tensorflow as tf  # lgtm [py/repeated-import]
 
-        diameter = self.patch_shape[0]
+        diameter = np.minimum(self.patch_shape[self.i_h], self.patch_shape[self.i_w])
 
         x = np.linspace(-1, 1, diameter)
         y = np.linspace(-1, 1, diameter)
@@ -202,19 +207,19 @@ def _get_circular_patch_mask(self, nb_images: int, sharpness: int = 40) -> "tf.T
         image_mask = 1 - np.clip(z_grid, -1, 1)
         image_mask = np.expand_dims(image_mask, axis=2)
         image_mask = np.broadcast_to(image_mask, self.patch_shape)
-        image_mask = tf.stack([image_mask] * nb_images)
+        image_mask = tf.stack([image_mask] * nb_samples)
         return image_mask
 
     def _random_overlay(self, images: np.ndarray, patch: np.ndarray, scale: Optional[float] = None) -> "tf.Tensor":
         import tensorflow as tf  # lgtm [py/repeated-import]
         import tensorflow_addons as tfa
 
-        nb_images = images.shape[0]
+        nb_samples = images.shape[0]
 
-        image_mask = self._get_circular_patch_mask(nb_images=nb_images)
+        image_mask = self._get_circular_patch_mask(nb_samples=nb_samples)
         image_mask = tf.cast(image_mask, images.dtype)
 
-        smallest_image_edge = np.minimum(self.image_shape[0], self.image_shape[1])
+        smallest_image_edge = np.minimum(self.image_shape[self.i_h], self.image_shape[self.i_w])
 
         image_mask = tf.image.resize(
             image_mask,
@@ -225,9 +230,15 @@ def _random_overlay(self, images: np.ndarray, patch: np.ndarray, scale: Optional
             name=None,
         )
 
+        pad_h_before = int((self.image_shape[self.i_h] - image_mask.shape[self.i_h + 1]) / 2)
+        pad_h_after = int(self.image_shape[self.i_h] - pad_h_before - image_mask.shape[self.i_h + 1])
+
+        pad_w_before = int((self.image_shape[self.i_w] - image_mask.shape[self.i_w + 1]) / 2)
+        pad_w_after = int(self.image_shape[self.i_w] - pad_w_before - image_mask.shape[self.i_w + 1])
+
         image_mask = tf.pad(
             image_mask,
-            paddings=tf.constant([[0, 0,], [0, 0,], [100, 100], [0, 0]]),
+            paddings=tf.constant([[0, 0], [pad_h_before, pad_h_after], [pad_w_before, pad_w_after], [0, 0]]),
             mode="CONSTANT",
             constant_values=0,
             name=None,
@@ -236,7 +247,7 @@ def _random_overlay(self, images: np.ndarray, patch: np.ndarray, scale: Optional
         image_mask = tf.cast(image_mask, images.dtype)
 
         patch = tf.cast(patch, images.dtype)
-        padded_patch = tf.stack([patch] * nb_images)
+        padded_patch = tf.stack([patch] * nb_samples)
 
         padded_patch = tf.image.resize(
             padded_patch,
@@ -247,15 +258,9 @@ def _random_overlay(self, images: np.ndarray, patch: np.ndarray, scale: Optional
             name=None,
         )
 
-        pad_w_0 = int((self.image_shape[0] - smallest_image_edge) / 2)
-        pad_w_1 = self.image_shape[0] - smallest_image_edge - pad_w_0
-
-        pad_h_0 = int((self.image_shape[1] - smallest_image_edge) / 2)
-        pad_h_1 = self.image_shape[1] - smallest_image_edge - pad_h_0
-
         padded_patch = tf.pad(
             padded_patch,
-            paddings=tf.constant([[0, 0], [pad_h_0, pad_h_1], [pad_w_0, pad_w_1], [0, 0]]),
+            paddings=tf.constant([[0, 0], [pad_h_before, pad_h_after], [pad_w_before, pad_w_after], [0, 0]]),
             mode="CONSTANT",
             constant_values=0,
             name=None,
@@ -265,14 +270,14 @@ def _random_overlay(self, images: np.ndarray, patch: np.ndarray, scale: Optional
 
         transform_vectors = list()
 
-        for i in range(nb_images):
+        for i in range(nb_samples):
             if scale is None:
                 im_scale = np.random.uniform(low=self.scale_min, high=self.scale_max)
             else:
                 im_scale = scale
 
-            padding_after_scaling_h = self.image_shape[0] - im_scale * padded_patch.shape[0]
-            padding_after_scaling_w = self.image_shape[1] - im_scale * padded_patch.shape[1]
+            padding_after_scaling_h = self.image_shape[self.i_h] - im_scale * padded_patch.shape[self.i_h]
+            padding_after_scaling_w = self.image_shape[self.i_w] - im_scale * padded_patch.shape[self.i_w]
 
             x_shift = np.random.uniform(-padding_after_scaling_w, padding_after_scaling_w)
             y_shift = np.random.uniform(-padding_after_scaling_h, padding_after_scaling_h)
@@ -336,7 +341,7 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> T
 
         return (
             self._patch.numpy(),
-            self._get_circular_patch_mask(nb_images=1).numpy()[0],
+            self._get_circular_patch_mask(nb_samples=1).numpy()[0],
         )
 
     def apply_patch(self, x: np.ndarray, scale: float, patch_external: Optional[np.ndarray] = None) -> np.ndarray: