Merge remote-tracking branch 'origin/dev_detection_transformer' into dev_detection_transformer

Author: kieranfraser

.github/workflows/dockerhub.yml

@@ -24,22 +24,22 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
+        uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@c4ee3adeed93b1fa6a762f209fb01608c1a22f1e
+        uses: docker/metadata-action@818d4b7b91585d195f67373fd9cb0332e31a7175
         with:
           images: adversarialrobustnesstoolbox/releases
           tags: |
             type=raw,value={{branch}}-1.14.1-{{sha}}
             type=semver,pattern={{version}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@3b5e8027fcad23fda98b2e3ac259d8d67585f671
+        uses: docker/build-push-action@2eb1c1961a95fc15694676618e422e8ba1d63825
         with:
           context: .
           push: true

art/attacks/attack.py

@@ -339,10 +339,10 @@ def __init__(self):
     @abc.abstractmethod
     def poison(
         self,
-        x: np.ndarray,
+        x: Union[np.ndarray, List[np.ndarray]],
         y: List[Dict[str, np.ndarray]],
         **kwargs,
-    ) -> Tuple[np.ndarray, List[Dict[str, np.ndarray]]]:
+    ) -> Tuple[Union[np.ndarray, List[np.ndarray]], List[Dict[str, np.ndarray]]]:
         """
         Generate poisoning examples and return them as an array. This method should be overridden by all concrete
         poisoning attack implementations.

art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py

@@ -123,7 +123,9 @@ def __init__(
 
         torch_version = list(map(int, torch.__version__.lower().split("+", maxsplit=1)[0].split(".")))
         torchvision_version = list(map(int, torchvision.__version__.lower().split("+", maxsplit=1)[0].split(".")))
-        assert torch_version[0] >= 1 and torch_version[1] >= 7, "AdversarialPatchPyTorch requires torch>=1.7.0"
+        assert (
+            torch_version[0] >= 1 and torch_version[1] >= 7 or (torch_version[0] >= 2)
+        ), "AdversarialPatchPyTorch requires torch>=1.7.0"
         assert (
             torchvision_version[0] >= 0 and torchvision_version[1] >= 8
         ), "AdversarialPatchPyTorch requires torchvision>=0.8.0"

art/attacks/poisoning/bad_det/bad_det_gma.py

@@ -23,7 +23,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 import numpy as np
 from tqdm.auto import tqdm
@@ -77,36 +77,39 @@ def __init__(
 
     def poison(  # pylint: disable=W0221
         self,
-        x: np.ndarray,
+        x: Union[np.ndarray, List[np.ndarray]],
         y: List[Dict[str, np.ndarray]],
         **kwargs,
-    ) -> Tuple[np.ndarray, List[Dict[str, np.ndarray]]]:
+    ) -> Tuple[Union[np.ndarray, List[np.ndarray]], List[Dict[str, np.ndarray]]]:
         """
         Generate poisoning examples by inserting the backdoor onto the input `x` and changing the classification
         for labels `y`.
 
-        :param x: Sample images of shape `NCHW` or `NHWC`.
+        :param x: Sample images of shape `NCHW` or `NHWC` or a list of sample images of any size.
         :param y: True labels of type `List[Dict[np.ndarray]]`, one dictionary per input image. The keys and values
                   of the dictionary are:
 
                   - boxes [N, 4]: the boxes in [x1, y1, x2, y2] format, with 0 <= x1 < x2 <= W and 0 <= y1 < y2 <= H.
                   - labels [N]: the labels for each image.
-                  - scores [N]: the scores or each prediction.
         :return: An tuple holding the `(poisoning_examples, poisoning_labels)`.
         """
-        x_ndim = len(x.shape)
+        if isinstance(x, np.ndarray):
+            x_ndim = len(x.shape)
+        else:
+            x_ndim = len(x[0].shape) + 1
 
         if x_ndim != 4:
             raise ValueError("Unrecognized input dimension. BadDet GMA can only be applied to image data.")
 
-        if self.channels_first:
-            # NCHW --> NHWC
-            x = np.transpose(x, (0, 2, 3, 1))
-
-        x_poison = x.copy()
-        y_poison: List[Dict[str, np.ndarray]] = []
+        # copy images
+        x_poison: Union[np.ndarray, List[np.ndarray]]
+        if isinstance(x, np.ndarray):
+            x_poison = x.copy()
+        else:
+            x_poison = [x_i.copy() for x_i in x]
 
         # copy labels
+        y_poison: List[Dict[str, np.ndarray]] = []
         for y_i in y:
             target_dict = {k: v.copy() for k, v in y_i.items()}
             y_poison.append(target_dict)
@@ -120,18 +123,22 @@ def poison(  # pylint: disable=W0221
             image = x_poison[i]
             labels = y_poison[i]["labels"]
 
+            if self.channels_first:
+                image = np.transpose(image, (1, 2, 0))
+
             # insert backdoor into the image
             # add an additional dimension to create a batch of size 1
             poisoned_input, _ = self.backdoor.poison(image[np.newaxis], labels)
-            x_poison[i] = poisoned_input[0]
+            image = poisoned_input[0]
+
+            # replace the original image with the poisoned image
+            if self.channels_first:
+                image = np.transpose(image, (2, 0, 1))
+            x_poison[i] = image
 
             # change all labels to the target label
             y_poison[i]["labels"] = np.full(labels.shape, self.class_target)
 
-        if self.channels_first:
-            # NHWC --> NCHW
-            x_poison = np.transpose(x_poison, (0, 3, 1, 2))
-
         return x_poison, y_poison
 
     def _check_params(self) -> None:

art/attacks/poisoning/bad_det/bad_det_oda.py

@@ -23,7 +23,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 import numpy as np
 from tqdm.auto import tqdm
@@ -77,36 +77,39 @@ def __init__(
 
     def poison(  # pylint: disable=W0221
         self,
-        x: np.ndarray,
+        x: Union[np.ndarray, List[np.ndarray]],
         y: List[Dict[str, np.ndarray]],
         **kwargs,
-    ) -> Tuple[np.ndarray, List[Dict[str, np.ndarray]]]:
+    ) -> Tuple[Union[np.ndarray, List[np.ndarray]], List[Dict[str, np.ndarray]]]:
         """
         Generate poisoning examples by inserting the backdoor onto the input `x` and changing the classification
         for labels `y`.
 
-        :param x: Sample images of shape `NCHW` or `NHWC`.
+        :param x: Sample images of shape `NCHW` or `NHWC` or a list of sample images of any size.
         :param y: True labels of type `List[Dict[np.ndarray]]`, one dictionary per input image. The keys and values
                   of the dictionary are:
 
                   - boxes [N, 4]: the boxes in [x1, y1, x2, y2] format, with 0 <= x1 < x2 <= W and 0 <= y1 < y2 <= H.
                   - labels [N]: the labels for each image.
-                  - scores [N]: the scores or each prediction.
         :return: An tuple holding the `(poisoning_examples, poisoning_labels)`.
         """
-        x_ndim = len(x.shape)
+        if isinstance(x, np.ndarray):
+            x_ndim = len(x.shape)
+        else:
+            x_ndim = len(x[0].shape) + 1
 
         if x_ndim != 4:
             raise ValueError("Unrecognized input dimension. BadDet ODA can only be applied to image data.")
 
-        if self.channels_first:
-            # NCHW --> NHWC
-            x = np.transpose(x, (0, 2, 3, 1))
-
-        x_poison = x.copy()
-        y_poison: List[Dict[str, np.ndarray]] = []
+        # copy images
+        x_poison: Union[np.ndarray, List[np.ndarray]]
+        if isinstance(x, np.ndarray):
+            x_poison = x.copy()
+        else:
+            x_poison = [x_i.copy() for x_i in x]
 
         # copy labels and find indices of the source class
+        y_poison: List[Dict[str, np.ndarray]] = []
         source_indices = []
         for i, y_i in enumerate(y):
             target_dict = {k: v.copy() for k, v in y_i.items()}
@@ -121,10 +124,12 @@ def poison(  # pylint: disable=W0221
 
         for i in tqdm(selected_indices, desc="BadDet ODA iteration", disable=not self.verbose):
             image = x_poison[i]
-
             boxes = y_poison[i]["boxes"]
             labels = y_poison[i]["labels"]
 
+            if self.channels_first:
+                image = np.transpose(image, (1, 2, 0))
+
             keep_indices = []
 
             for j, (box, label) in enumerate(zip(boxes, labels)):
@@ -140,13 +145,14 @@ def poison(  # pylint: disable=W0221
                 else:
                     keep_indices.append(j)
 
+            # replace the original image with the poisoned image
+            if self.channels_first:
+                image = np.transpose(image, (2, 0, 1))
+            x_poison[i] = image
+
             # remove labels for poisoned bounding boxes
             y_poison[i] = {k: v[keep_indices] for k, v in y_poison[i].items()}
 
-        if self.channels_first:
-            # NHWC --> NCHW
-            x_poison = np.transpose(x_poison, (0, 3, 1, 2))
-
         return x_poison, y_poison
 
     def _check_params(self) -> None:

art/attacks/poisoning/bad_det/bad_det_oga.py

@@ -23,7 +23,7 @@
 from __future__ import absolute_import, division, print_function, unicode_literals
 
 import logging
-from typing import Dict, List, Tuple
+from typing import Dict, List, Tuple, Union
 
 import numpy as np
 from tqdm.auto import tqdm
@@ -85,35 +85,39 @@ def __init__(
 
     def poison(  # pylint: disable=W0221
         self,
-        x: np.ndarray,
+        x: Union[np.ndarray, List[np.ndarray]],
         y: List[Dict[str, np.ndarray]],
         **kwargs,
-    ) -> Tuple[np.ndarray, List[Dict[str, np.ndarray]]]:
+    ) -> Tuple[Union[np.ndarray, List[np.ndarray]], List[Dict[str, np.ndarray]]]:
         """
         Generate poisoning examples by inserting the backdoor onto the input `x` and changing the classification
         for labels `y`.
 
-        :param x: Sample images of shape `NCHW` or `NHWC`.
+        :param x: Sample images of shape `NCHW` or `NHWC` or a list of sample images of any size.
         :param y: True labels of type `List[Dict[np.ndarray]]`, one dictionary per input image. The keys and values
                   of the dictionary are:
+
                   - boxes [N, 4]: the boxes in [x1, y1, x2, y2] format, with 0 <= x1 < x2 <= W and 0 <= y1 < y2 <= H.
                   - labels [N]: the labels for each image.
-                  - scores [N]: the scores or each prediction.
         :return: An tuple holding the `(poisoning_examples, poisoning_labels)`.
         """
-        x_ndim = len(x.shape)
+        if isinstance(x, np.ndarray):
+            x_ndim = len(x.shape)
+        else:
+            x_ndim = len(x[0].shape) + 1
 
         if x_ndim != 4:
             raise ValueError("Unrecognized input dimension. BadDet OGA can only be applied to image data.")
 
-        if self.channels_first:
-            # NCHW --> NHWC
-            x = np.transpose(x, (0, 2, 3, 1))
-
-        x_poison = x.copy()
-        y_poison: List[Dict[str, np.ndarray]] = []
+        # copy images
+        x_poison: Union[np.ndarray, List[np.ndarray]]
+        if isinstance(x, np.ndarray):
+            x_poison = x.copy()
+        else:
+            x_poison = [x_i.copy() for x_i in x]
 
         # copy labels
+        y_poison: List[Dict[str, np.ndarray]] = []
         for y_i in y:
             target_dict = {k: v.copy() for k, v in y_i.items()}
             y_poison.append(target_dict)
@@ -123,14 +127,15 @@ def poison(  # pylint: disable=W0221
         num_poison = int(self.percent_poison * len(all_indices))
         selected_indices = np.random.choice(all_indices, num_poison, replace=False)
 
-        _, height, width, _ = x_poison.shape
-
         for i in tqdm(selected_indices, desc="BadDet OGA iteration", disable=not self.verbose):
             image = x_poison[i]
-
             boxes = y_poison[i]["boxes"]
             labels = y_poison[i]["labels"]
 
+            if self.channels_first:
+                image = np.transpose(image, (1, 2, 0))
+            height, width, _ = image.shape
+
             # generate the fake bounding box
             y_1 = np.random.randint(0, height - self.bbox_height)
             x_1 = np.random.randint(0, width - self.bbox_width)
@@ -145,6 +150,11 @@ def poison(  # pylint: disable=W0221
             poisoned_input, _ = self.backdoor.poison(bounding_box[np.newaxis], labels)
             image[y_1:y_2, x_1:x_2, :] = poisoned_input[0]
 
+            # replace the original image with the poisoned image
+            if self.channels_first:
+                image = np.transpose(image, (2, 0, 1))
+            x_poison[i] = image
+
             # insert the fake bounding box and label
             y_poison[i]["boxes"] = np.concatenate((boxes, [[x_1, y_1, x_2, y_2]]))
             y_poison[i]["labels"] = np.concatenate((labels, [self.class_target]))
@@ -155,10 +165,6 @@ def poison(  # pylint: disable=W0221
                 mask[y_1:y_2, x_1:x_2, :] = 1
                 y_poison[i]["masks"] = np.concatenate((y_poison[i]["masks"], [mask]))
 
-        if self.channels_first:
-            # NHWC --> NCHW
-            x_poison = np.transpose(x_poison, (0, 3, 1, 2))
-
         return x_poison, y_poison
 
     def _check_params(self) -> None: