Merge pull request #2169 from kieranfraser/dev_issue_2148_yolo

Fix AdversarialPatchPyTorch compatibility with YOLO estimator

Author: beat-buesser

art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py

@@ -575,9 +575,9 @@ def __getitem__(self, idx):
                     img = torch.from_numpy(self.x[idx])
 
                     target = {}
-                    target["boxes"] = torch.from_numpy(y[idx]["boxes"])
-                    target["labels"] = torch.from_numpy(y[idx]["labels"])
-                    target["scores"] = torch.from_numpy(y[idx]["scores"])
+                    target["boxes"] = torch.from_numpy(self.y[idx]["boxes"])
+                    target["labels"] = torch.from_numpy(self.y[idx]["labels"])
+                    target["scores"] = torch.from_numpy(self.y[idx]["scores"])
                     mask_i = torch.from_numpy(self.mask[idx])
 
                     return img, target, mask_i
@@ -602,19 +602,33 @@ def __getitem__(self, idx):
                     if isinstance(target, torch.Tensor):
                         target = target.to(self.estimator.device)
                     else:
-                        target["boxes"] = target["boxes"].to(self.estimator.device)
-                        target["labels"] = target["labels"].to(self.estimator.device)
-                        target["scores"] = target["scores"].to(self.estimator.device)
+                        targets = []
+                        for idx in range(target["boxes"].shape[0]):
+                            targets.append(
+                                {
+                                    "boxes": target["boxes"][idx].to(self.estimator.device),
+                                    "labels": target["labels"][idx].to(self.estimator.device),
+                                    "scores": target["scores"][idx].to(self.estimator.device),
+                                }
+                            )
+                        target = targets
                     _ = self._train_step(images=images, target=target, mask=None)
             else:
                 for images, target, mask_i in data_loader:
                     images = images.to(self.estimator.device)
                     if isinstance(target, torch.Tensor):
                         target = target.to(self.estimator.device)
                     else:
-                        target["boxes"] = target["boxes"].to(self.estimator.device)
-                        target["labels"] = target["labels"].to(self.estimator.device)
-                        target["scores"] = target["scores"].to(self.estimator.device)
+                        targets = []
+                        for idx in range(target["boxes"].shape[0]):
+                            targets.append(
+                                {
+                                    "boxes": target["boxes"][idx].to(self.estimator.device),
+                                    "labels": target["labels"][idx].to(self.estimator.device),
+                                    "scores": target["scores"][idx].to(self.estimator.device),
+                                }
+                            )
+                        target = targets
                     mask_i = mask_i.to(self.estimator.device)
                     _ = self._train_step(images=images, target=target, mask=mask_i)
 

art/estimators/object_detection/pytorch_yolo.py

@@ -294,7 +294,10 @@ def _preprocess_and_convert_inputs(
 
             # Set gradients
             if not no_grad:
-                x_tensor.requires_grad = True
+                if x_tensor.is_leaf:
+                    x_tensor.requires_grad = True
+                else:
+                    x_tensor.retain_grad()
 
             # Apply framework-specific preprocessing
             x_preprocessed, y_preprocessed = self._apply_preprocessing(x=x_tensor, y=y_tensor, fit=fit, no_grad=no_grad)

notebooks/adversarial_patch/attack_adversarial_patch_pytorch_yolo.ipynb

@@ -367,7 +367,7 @@ def test_compute_loss(art_warning, get_pytorch_yolo):
         # Compute loss
         loss = object_detector.compute_loss(x=x_test, y=y_test)
 
-        assert pytest.approx(11.20741, abs=0.9) == float(loss)
+        assert pytest.approx(11.20741, abs=1.5) == float(loss)
 
     except ARTTestException as e:
         art_warning(e)
@@ -386,3 +386,66 @@ def test_pgd(art_warning, get_pytorch_yolo):
 
     except ARTTestException as e:
         art_warning(e)
+
+
+@pytest.mark.only_with_platform("pytorch")
+def test_patch(art_warning, get_pytorch_yolo):
+    try:
+
+        from art.attacks.evasion import AdversarialPatchPyTorch
+
+        rotation_max = 0.0
+        scale_min = 0.1
+        scale_max = 0.3
+        distortion_scale_max = 0.0
+        learning_rate = 1.99
+        max_iter = 2
+        batch_size = 16
+        patch_shape = (3, 5, 5)
+        patch_type = "circle"
+        optimizer = "pgd"
+
+        object_detector, x_test, y_test = get_pytorch_yolo
+
+        ap = AdversarialPatchPyTorch(
+            estimator=object_detector,
+            rotation_max=rotation_max,
+            scale_min=scale_min,
+            scale_max=scale_max,
+            optimizer=optimizer,
+            distortion_scale_max=distortion_scale_max,
+            learning_rate=learning_rate,
+            max_iter=max_iter,
+            batch_size=batch_size,
+            patch_shape=patch_shape,
+            patch_type=patch_type,
+            verbose=True,
+            targeted=False,
+        )
+
+        _, _ = ap.generate(x=x_test, y=y_test)
+
+        patched_images = ap.apply_patch(x_test, scale=0.4)
+        result = object_detector.predict(patched_images)
+
+        assert result[0]["scores"].shape == (10647,)
+        expected_detection_scores = np.asarray(
+            [
+                4.3653536e-08,
+                3.3987994e-06,
+                2.5681820e-06,
+                3.9782722e-06,
+                2.1766680e-05,
+                2.6138965e-05,
+                6.3377396e-05,
+                7.6248516e-06,
+                4.3447722e-06,
+                3.6515078e-06,
+            ]
+        )
+        np.testing.assert_raises(
+            AssertionError, np.testing.assert_array_almost_equal, result[0]["scores"][:10], expected_detection_scores, 6
+        )
+
+    except ARTTestException as e:
+        art_warning(e)