Merge pull request #1693 from Trusted-AI/development_issue_1692

Update RobustDPatch for channels first images

Author: beat-buesser

art/attacks/evasion/dpatch_robust.py

@@ -371,6 +371,8 @@ def _augment_images_with_patch(
         if self.targeted:
             predictions = y_copy
         else:
+            if channels_first:
+                x_copy = np.transpose(x_copy, (0, 3, 1, 2))
             predictions = self.estimator.predict(x=x_copy, standardise_output=True)
 
         for i_image in range(x_copy.shape[0]):
@@ -413,8 +415,12 @@ def _untransform_gradients(
         # Account for cropping when considering the upper left point of the patch:
         x_1 = self.patch_location[0] - int(transforms["crop_x"])
         y_1 = self.patch_location[1] - int(transforms["crop_y"])
-        x_2 = x_1 + self.patch_shape[0]
-        y_2 = y_1 + self.patch_shape[1]
+        if channels_first:
+            x_2 = x_1 + self.patch_shape[1]
+            y_2 = y_1 + self.patch_shape[2]
+        else:
+            x_2 = x_1 + self.patch_shape[0]
+            y_2 = y_1 + self.patch_shape[1]
         gradients = gradients[:, x_1:x_2, y_1:y_2, :]
 
         if channels_first:

art/estimators/object_detection/pytorch_faster_rcnn.py

@@ -70,7 +70,7 @@ def __init__(
                maximum values allowed for features. If floats are provided, these will be used as the range of all
                features. If arrays are provided, each value will be considered the bound for a feature, thus
                the shape of clip values needs to match the total number of features.
-        :param channels_first: [Currently unused] Set channels first or last.
+        :param channels_first: Set channels first or last.
         :param preprocessing_defences: Preprocessing defence(s) to be applied by the classifier.
         :param postprocessing_defences: Postprocessing defence(s) to be applied by the classifier.
         :param preprocessing: Tuple of the form `(subtrahend, divisor)` of floats or `np.ndarray` of values to be

art/estimators/object_detection/pytorch_object_detector.py

@@ -74,7 +74,7 @@ def __init__(
                maximum values allowed for features. If floats are provided, these will be used as the range of all
                features. If arrays are provided, each value will be considered the bound for a feature, thus
                the shape of clip values needs to match the total number of features.
-        :param channels_first: [Currently unused] Set channels first or last.
+        :param channels_first: Set channels first or last.
         :param preprocessing_defences: Preprocessing defence(s) to be applied by the classifier.
         :param postprocessing_defences: Postprocessing defence(s) to be applied by the classifier.
         :param preprocessing: Tuple of the form `(subtrahend, divisor)` of floats or `np.ndarray` of values to be

tests/attacks/evasion/test_dpatch_robust.py

@@ -107,9 +107,9 @@ def test_generate_targeted(art_warning, fix_get_mnist_subset, fix_get_rcnn, fram
         art_warning(e)
 
 
-@pytest.mark.parametrize("image_format", ["NHWC", "NCHW"])
+@pytest.mark.parametrize("channels_first", [False, True])
 @pytest.mark.skip_framework("keras", "scikitlearn", "mxnet", "kerastf")
-def test_augment_images_with_patch(art_warning, image_format, fix_get_rcnn):
+def test_augment_images_with_patch(art_warning, channels_first, fix_get_rcnn):
     try:
         frcnn = fix_get_rcnn
         attack = RobustDPatch(
@@ -126,14 +126,12 @@ def test_augment_images_with_patch(art_warning, image_format, fix_get_rcnn):
             verbose=False,
         )
 
-        if image_format == "NHWC":
-            patch = np.ones(shape=(4, 4, 1))
-            x = np.zeros(shape=(1, 10, 10, 1))
-            channels_first = False
-        elif image_format == "NCHW":
+        if channels_first:
             patch = np.ones(shape=(1, 4, 4))
             x = np.zeros(shape=(1, 1, 10, 10))
-            channels_first = True
+        else:
+            patch = np.ones(shape=(4, 4, 1))
+            x = np.zeros(shape=(1, 10, 10, 1))
 
         patched_images, _, transformations = attack._augment_images_with_patch(
             x=x, y=None, patch=patch, channels_first=channels_first
@@ -143,10 +141,10 @@ def test_augment_images_with_patch(art_warning, image_format, fix_get_rcnn):
         patch_sum_expected = 16.0
         complement_sum_expected = 0.0
 
-        if image_format == "NHWC":
-            patch_sum = np.sum(patched_images[0, 2:7, 2:7, :])
-        elif image_format == "NCHW":
+        if channels_first:
             patch_sum = np.sum(patched_images[0, :, 2:7, 2:7])
+        else:
+            patch_sum = np.sum(patched_images[0, 2:7, 2:7, :])
 
         complement_sum = np.sum(patched_images[0]) - patch_sum
 
@@ -196,13 +194,32 @@ def test_apply_patch(art_warning, fix_get_rcnn):
         art_warning(e)
 
 
+@pytest.mark.parametrize("channels_first", [False, True])
 @pytest.mark.skip_framework("keras", "scikitlearn", "mxnet", "kerastf")
-def test_untransform_gradients(art_warning, fix_get_rcnn):
+def test_untransform_gradients(art_warning, fix_get_rcnn, channels_first):
     try:
+        crop_x = 1
+        crop_y = 1
+        rot90 = 3
+        brightness = 1.0
+
+        if channels_first:
+            patch_shape = (1, 4, 4)
+            gradients = np.zeros(shape=(1, 1, 10, 10))
+            gradients[:, :, 2:7, 2:7] = 1
+            gradients = gradients[:, :, crop_x : 10 - crop_x, crop_y : 10 - crop_y]
+            gradients = np.rot90(gradients, rot90, (2, 3))
+        else:
+            patch_shape = (4, 4, 1)
+            gradients = np.zeros(shape=(1, 10, 10, 1))
+            gradients[:, 2:7, 2:7, :] = 1
+            gradients = gradients[:, crop_x : 10 - crop_x, crop_y : 10 - crop_y, :]
+            gradients = np.rot90(gradients, rot90, (1, 2))
+
         frcnn = fix_get_rcnn
         attack = RobustDPatch(
             frcnn,
-            patch_shape=(4, 4, 1),
+            patch_shape=patch_shape,
             patch_location=(2, 2),
             crop_range=(0, 0),
             brightness_range=(1.0, 1.0),
@@ -214,20 +231,12 @@ def test_untransform_gradients(art_warning, fix_get_rcnn):
             verbose=False,
         )
 
-        gradients = np.zeros(shape=(1, 10, 10, 1))
-        gradients[:, 2:7, 2:7, :] = 1
-
-        crop_x = 1
-        crop_y = 1
-        rot90 = 3
-        brightness = 1.0
-
-        gradients = gradients[:, crop_x : 10 - crop_x, crop_y : 10 - crop_y, :]
-        gradients = np.rot90(gradients, rot90, (1, 2))
-
         transforms = {"crop_x": crop_x, "crop_y": crop_y, "rot90": rot90, "brightness": brightness}
 
-        gradients = attack._untransform_gradients(gradients=gradients, transforms=transforms, channels_first=False)
+        gradients = attack._untransform_gradients(
+            gradients=gradients, transforms=transforms, channels_first=channels_first
+        )
+
         gradients_sum = np.sum(gradients[0])
         gradients_sum_expected = 16.0