Merge pull request #1120 from Trusted-AI/dev_1.6.2

Update to ART 1.6.2

Author: beat-buesser

.github/workflows/ci-mxnet.yml

@@ -1,4 +1,4 @@
-name: CI MXNet
+name: CI MXNet + Legacy
 on:
   # Run on manual trigger
   workflow_dispatch:

.github/workflows/ci-style-checks.yml

@@ -0,0 +1,60 @@
+name: CI Style Checks
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  style:
+    name: Style Check
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Setup Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.7
+      - name: Pre-install
+        run: |
+          sudo apt-get update
+          sudo apt-get -y -q install ffmpeg libavcodec-extra
+          pip install tensorflow==2.4.1
+          pip install keras==2.4.3
+      - name: Install Dependencies
+        run: |
+          python -m pip install --upgrade pip setuptools wheel
+          pip install -q pylint==2.7.4 mypy==0.812 pycodestyle==2.7.0 black==20.8b1
+          pip install -q -r requirements.txt
+          pip list
+      - name: pycodestyle
+        run: pycodestyle --ignore=C0330,C0415,E203,E231,W503 --max-line-length=120 art
+      - name: pylint
+        if: ${{ always() }}
+        run: pylint --disable=C0330,C0415,E203,E1136,E0401,E1102 -rn art
+      - name: mypy
+        if: ${{ always() }}
+        run: mypy art
+      - name: pytest-flake8
+        if: ${{ always() }}
+        run: pytest --flake8 -v -m flake8
+      - name: black
+        if: ${{ always() }}
+        run: |
+          black --line-length 120 --check art/
+          black --line-length 120 --check tests/
+          black --line-length 120 --check examples/

.github/workflows/ci.yml

@@ -134,36 +134,3 @@ jobs:
         run: ./run_tests.sh ${{ matrix.framework }}
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]
-  style:
-    name: Style Check
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout Repo
-        uses: actions/[email protected]
-      - name: Setup Python
-        uses: actions/[email protected]
-        with:
-          python-version: 3.7
-      - name: Pre-install
-        run: |
-          sudo apt-get update
-          sudo apt-get -y -q install ffmpeg libavcodec-extra
-          pip install tensorflow==2.4.1
-          pip install keras==2.4.3
-      - name: Install Dependencies
-        run: |
-          python -m pip install --upgrade pip setuptools wheel
-          pip install -q pylint==2.7.4 pycodestyle==2.7.0
-          pip install -q -r requirements.txt
-          pip list
-      - name: pycodestyle
-        run: pycodestyle --ignore=C0330,C0415,E203,E231,W503 --max-line-length=120 art
-      - name: pylint
-        if: ${{ always() }}
-        run: pylint --disable=C0330,C0415,E203,E1136,E0401,E1102 -rn art
-      - name: mypy
-        if: ${{ always() }}
-        run: mypy art
-      - name: pytest-flake8
-        if: ${{ always() }}
-        run: pytest --flake8 -v -m flake8

art/attacks/evasion/adversarial_patch/adversarial_patch_pytorch.py

@@ -100,8 +100,8 @@ def __init__(
         import torch  # lgtm [py/repeated-import]
         import torchvision
 
-        torch_version = list(map(int, torch.__version__.lower().split(".")))
-        torchvision_version = list(map(int, torchvision.__version__.lower().split(".")))
+        torch_version = list(map(int, torch.__version__.lower().split("+")[0].split(".")))
+        torchvision_version = list(map(int, torchvision.__version__.lower().split("+")[0].split(".")))
         assert torch_version[0] >= 1 and torch_version[1] >= 7, "AdversarialPatchPyTorch requires torch>=1.7.0"
         assert (
             torchvision_version[0] >= 0 and torchvision_version[1] >= 8

art/attacks/evasion/dpatch.py

@@ -189,7 +189,7 @@ def generate(  # pylint: disable=W0221
 
         else:
 
-            predictions = self.estimator.predict(x=patched_images)
+            predictions = self.estimator.predict(x=patched_images, standardise_output=True)
 
             for i_image in range(patched_images.shape[0]):
                 target_dict = dict()
@@ -213,6 +213,7 @@ def generate(  # pylint: disable=W0221
                 gradients = self.estimator.loss_gradient(
                     x=patched_images[i_batch_start:i_batch_end],
                     y=patch_target[i_batch_start:i_batch_end],
+                    standardise_output=True,
                 )
 
                 for i_image in range(gradients.shape[0]):

art/attacks/evasion/dpatch_robust.py

@@ -59,12 +59,13 @@ class RobustDPatch(EvasionAttack):
         "learning_rate",
         "max_iter",
         "batch_size",
-        "verbose",
         "patch_location",
         "crop_range",
         "brightness_range",
         "rotation_weights",
         "sample_size",
+        "targeted",
+        "verbose",
     ]
 
     _estimator_requirements = (BaseEstimator, LossGradientsMixin, ObjectDetectorMixin)
@@ -81,6 +82,7 @@ def __init__(
         learning_rate: float = 5.0,
         max_iter: int = 500,
         batch_size: int = 16,
+        targeted: bool = False,
         verbose: bool = True,
     ):
         """
@@ -96,6 +98,7 @@ def __init__(
         :param learning_rate: The learning rate of the optimization.
         :param max_iter: The number of optimization steps.
         :param batch_size: The size of the training batch.
+        :param targeted: Indicates whether the attack is targeted (True) or untargeted (False).
         :param verbose: Show progress bars.
         """
 
@@ -120,9 +123,10 @@ def __init__(
         self.brightness_range = brightness_range
         self.rotation_weights = rotation_weights
         self.sample_size = sample_size
+        self._targeted = targeted
         self._check_params()
 
-    def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
+    def generate(self, x: np.ndarray, y: Optional[List[Dict[str, np.ndarray]]] = None, **kwargs) -> np.ndarray:
         """
         Generate RobustDPatch.
 
@@ -133,7 +137,9 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         channel_index = 1 if self.estimator.channels_first else x.ndim - 1
         if x.shape[channel_index] != self.patch_shape[channel_index - 1]:
             raise ValueError("The color channel index of the images and the patch have to be identical.")
-        if y is not None:
+        if y is None and self.targeted:
+            raise ValueError("The targeted version of RobustDPatch attack requires target labels provided to `y`.")
+        if y is not None and not self.targeted:
             raise ValueError("The RobustDPatch attack does not use target labels.")
         if x.ndim != 4:
             raise ValueError("The adversarial patch can only be applied to images.")
@@ -144,6 +150,24 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         else:
             image_height, image_width = x.shape[1:3]
 
+        if not self.estimator.native_label_is_pytorch_format and y is not None:
+            from art.estimators.object_detection.utils import convert_tf_to_pt
+
+            y = convert_tf_to_pt(y=y, height=x.shape[1], width=x.shape[2])
+
+        if y is not None:
+            for i_image in range(x.shape[0]):
+                y_i = y[i_image]["boxes"]
+                for i_box in range(y_i.shape[0]):
+                    x_1, y_1, x_2, y_2 = y_i[i_box]
+                    if (
+                        x_1 < self.crop_range[1]
+                        or y_1 < self.crop_range[0]
+                        or x_2 > image_width - self.crop_range[1] + 1
+                        or y_2 > image_height - self.crop_range[0] + 1
+                    ):
+                        raise ValueError("Cropping is intersecting with at least one box, reduce `crop_range`.")
+
         if (
             self.patch_location[0] + self.patch_shape[0] > image_height - self.crop_range[0]
             or self.patch_location[1] + self.patch_shape[1] > image_width - self.crop_range[1]
@@ -165,14 +189,20 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
                     i_batch_start = i_batch * self.batch_size
                     i_batch_end = min((i_batch + 1) * self.batch_size, x.shape[0])
 
+                    if y is None:
+                        y_batch = y
+                    else:
+                        y_batch = y[i_batch_start:i_batch_end]
+
                     # Sample and apply the random transformations:
                     patched_images, patch_target, transforms = self._augment_images_with_patch(
-                        x[i_batch_start:i_batch_end], self._patch, channels_first=self.estimator.channels_first
+                        x[i_batch_start:i_batch_end], y_batch, self._patch, channels_first=self.estimator.channels_first
                     )
 
                     gradients = self.estimator.loss_gradient(
                         x=patched_images,
                         y=patch_target,
+                        standardise_output=True,
                     )
 
                     gradients = self._untransform_gradients(
@@ -187,7 +217,7 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
 
                     patch_gradients_old = patch_gradients
 
-            self._patch = self._patch + np.sign(patch_gradients) * self.learning_rate
+            self._patch = self._patch + np.sign(patch_gradients) * (1 - 2 * int(self.targeted)) * self.learning_rate
 
             if self.estimator.clip_values is not None:
                 self._patch = np.clip(
@@ -199,12 +229,13 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         return self._patch
 
     def _augment_images_with_patch(
-        self, x: np.ndarray, patch: np.ndarray, channels_first: bool
+        self, x: np.ndarray, y: Optional[List[Dict[str, np.ndarray]]], patch: np.ndarray, channels_first: bool
     ) -> Tuple[np.ndarray, List[Dict[str, np.ndarray]], Dict[str, Union[int, float]]]:
         """
         Augment images with patch.
 
         :param x: Sample images.
+        :param y: Target labels.
         :param patch: The patch to be applied.
         :param channels_first: Set channels first or last.
         """
@@ -242,17 +273,73 @@ def _augment_images_with_patch(
 
         transformations.update({"rot90": rot90})
 
+        if y is not None:
+
+            y_copy: List[Dict[str, np.ndarray]] = list()
+
+            for i_image in range(x_copy.shape[0]):
+                y_b = y[i_image]["boxes"].copy()
+                image_width = x.shape[2]
+                image_height = x.shape[1]
+                x_1_arr = y_b[:, 0]
+                y_1_arr = y_b[:, 1]
+                x_2_arr = y_b[:, 2]
+                y_2_arr = y_b[:, 3]
+                box_width = x_2_arr - x_1_arr
+                box_height = y_2_arr - y_1_arr
+
+                if rot90 == 0:
+                    x_1_new = x_1_arr
+                    y_1_new = y_1_arr
+                    x_2_new = x_2_arr
+                    y_2_new = y_2_arr
+
+                if rot90 == 1:
+                    x_1_new = y_1_arr
+                    y_1_new = image_width - x_1_arr - box_width
+                    x_2_new = y_1_arr + box_height
+                    y_2_new = image_width - x_1_arr
+
+                if rot90 == 2:
+                    x_1_new = image_width - x_2_arr
+                    y_1_new = image_height - y_2_arr
+                    x_2_new = x_1_new + box_width
+                    y_2_new = y_1_new + box_height
+
+                if rot90 == 3:
+                    x_1_new = image_height - y_1_arr - box_height
+                    y_1_new = x_1_arr
+                    x_2_new = image_height - y_1_arr
+                    y_2_new = x_1_arr + box_width
+
+                y_i = dict()
+                y_i["boxes"] = np.zeros_like(y[i_image]["boxes"])
+                y_i["boxes"][:, 0] = x_1_new
+                y_i["boxes"][:, 1] = y_1_new
+                y_i["boxes"][:, 2] = x_2_new
+                y_i["boxes"][:, 3] = y_2_new
+
+                y_i["labels"] = y[i_image]["labels"]
+                y_i["scores"] = y[i_image]["scores"]
+
+                y_copy.append(y_i)
+
         # 3) adjust brightness:
         brightness = random.uniform(*self.brightness_range)
-        x_copy = np.round(brightness * x_copy)
-        x_patch = np.round(brightness * x_patch)
+        x_copy = np.round(brightness * x_copy / self.learning_rate) * self.learning_rate
+        x_patch = np.round(brightness * x_patch / self.learning_rate) * self.learning_rate
 
         transformations.update({"brightness": brightness})
 
         logger.debug("Transformations: %s", str(transformations))
 
         patch_target: List[Dict[str, np.ndarray]] = list()
-        predictions = self.estimator.predict(x=x_copy)
+
+        if self.targeted:
+            predictions = y_copy
+        else:
+            predictions = self.estimator.predict(x=x_copy, standardise_output=True)
+
         for i_image in range(x_copy.shape[0]):
             target_dict = dict()
             target_dict["boxes"] = predictions[i_image]["boxes"]
@@ -385,8 +472,8 @@ def _check_params(self) -> None:
         if len(self.brightness_range) != 2:
             raise ValueError("The length of brightness range must be 2.")
 
-        if self.brightness_range[0] < 0.0 or self.brightness_range[1] > 1.0:
-            raise ValueError("The brightness range must be between 0.0 and 1.0.")
+        if self.brightness_range[0] < 0.0:
+            raise ValueError("The brightness range must be >= 0.0.")
 
         if self.brightness_range[0] > self.brightness_range[1]:
             raise ValueError("The first element of the brightness range must be less or equal to the second one.")
@@ -408,3 +495,6 @@ def _check_params(self) -> None:
             raise ValueError("The EOT sample size must be of type int.")
         if self.sample_size <= 0:
             raise ValueError("The EOT sample size must be greater than 0.")
+
+        if not isinstance(self.targeted, bool):
+            raise ValueError("The argument `targeted` has to be of type bool.")

art/attacks/evasion/iterative_method.py

@@ -53,6 +53,7 @@ def __init__(
         max_iter: int = 100,
         targeted: bool = False,
         batch_size: int = 32,
+        verbose: bool = True,
     ) -> None:
         """
         Create a :class:`.ProjectedGradientDescent` instance.
@@ -63,6 +64,7 @@ def __init__(
         :param max_iter: The maximum number of iterations.
         :param targeted: Indicates whether the attack is targeted (True) or untargeted (False).
         :param batch_size: Size of the batch on which adversarial samples are generated.
+        :param verbose: Show progress bars.
         """
         super().__init__(
             estimator=estimator,
@@ -73,4 +75,5 @@ def __init__(
             targeted=targeted,
             num_random_init=0,
             batch_size=batch_size,
+            verbose=verbose,
         )

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_pytorch.py

@@ -293,7 +293,7 @@ def _compute_perturbation(  # pylint: disable=W0221
 
         # Apply mask
         if mask is not None:
-            grad = torch.where(mask == 0.0, torch.tensor(0.0), grad)
+            grad = torch.where(mask == 0.0, torch.tensor(0.0).to(self.estimator.device), grad)
 
         # Apply norm bound
         if self.norm in ["inf", np.inf]:

art/defences/detector/poison/ground_truth_evaluator.py

@@ -95,7 +95,7 @@ def analyze_correctness(
             dic_json.update({key_i: matrix_i})
             all_errors_by_class.append(errors)
 
-        all_errors_by_class = np.asarray(all_errors_by_class)
+        all_errors_by_class = np.asarray(all_errors_by_class, dtype=object)
         conf_matrix_json = json.dumps(dic_json)
 
         return all_errors_by_class, conf_matrix_json