Merge branch 'dev_1.21.0' into detector-refactoring

Author: beat-buesser

.github/actions/yolo/Dockerfile

@@ -24,4 +24,6 @@ RUN pip install pytorchyolo==1.8.0 tensorflow==2.14.1 scikit-learn==1.4.2 pytest
 
 RUN cd /tmp/ && git clone https://github.com/eriklindernoren/PyTorch-YOLOv3.git && cd ./PyTorch-YOLOv3/weights && ./download_weights.sh
 
+RUN mkdir /tmp/yolo_v8.3.0 && cd /tmp/yolo_v8.3.0 && wget https://github.com/ultralytics/assets/releases/download/v8.3.0/yolov8n.pt && wget https://github.com/ultralytics/assets/releases/download/v8.3.0/yolov10n.pt
+
 CMD ["/bin/bash"]

.github/workflows/dockerhub.yml

@@ -34,7 +34,7 @@ jobs:
         with:
           images: adversarialrobustnesstoolbox/releases
           tags: |
-            type=raw,value={{branch}}-1.19.2-{{sha}}
+            type=raw,value={{branch}}-1.20.1-{{sha}}
             type=semver,pattern={{version}}
 
       - name: Build and push Docker image

README-cn.md

@@ -1,6 +1,6 @@
-# Adversarial Robustness Toolbox (ART) v1.19
+# Adversarial Robustness Toolbox (ART) v1.20
 <p align="center">
-  <img src="docs/images/art_lfai.png?raw=true" width="467" title="ART logo">
+  <img src="https://raw.githubusercontent.com/Trusted-AI/adversarial-robustness-toolbox/main/docs/images/art_lfai.png" width="467" title="ART logo">
 </p>
 <br />
 
@@ -16,9 +16,14 @@
 [![Downloads](https://static.pepy.tech/badge/adversarial-robustness-toolbox/month)](https://pepy.tech/project/adversarial-robustness-toolbox)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/5090/badge)](https://bestpractices.coreinfrastructure.org/projects/5090)
 
-<p align="center">
-  <img src="https://raw.githubusercontent.com/lfai/artwork/master/lfaidata-assets/lfaidata-project-badge/graduate/color/lfaidata-project-badge-graduate-color.png" alt="LF AI & Data" width="300"/>
-</p>
+ <div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/images/lfaidata-project-badge-graduate-color_dark.png" width="400" title="LF AI & Data">
+    <source media="(prefers-color-scheme: light)" srcset="docs/images/lfaidata-project-badge-graduate-color.png" width="400" title="LF AI & Data">
+    <img alt="Fallback image description" src="default-image.png" width="400">
+  </picture>
+</div>
+<br />
 
 对抗性鲁棒性工具集（ART）是用于机器学习安全性的Python库。ART 由
 [Linux Foundation AI & Data Foundation](https://lfaidata.foundation) (LF AI & Data)。 ART提供的工具可
@@ -30,17 +35,28 @@
 
 ## Adversarial Threats
 
+ <div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/images/adversarial_threats_attacker_dark.png" width="400 title="ART Threats">
+    <source media="(prefers-color-scheme: light)" srcset="docs/images/adversarial_threats_attacker.png" width="400 title="ART Threats">
+    <img alt="Fallback image description" src="default-image.png" width="400">
+  </picture>
+</div>
+
 <p align="center">
-  <img src="docs/images/adversarial_threats_attacker.png?raw=true" width="400" title="ART logo">
-  <img src="docs/images/adversarial_threats_art.png?raw=true" width="400" title="ART logo">
+  <img src="docs/images/adversarial_threats_art.png?raw=true" width="400" title="ART Matrix">
 </p>
 <br />
 
 ## ART for Red and Blue Teams (selection)
 
-<p align="center">
-  <img src="docs/images/white_hat_blue_red.png?raw=true" width="800" title="ART Red and Blue Teams">
-</p>
+ <div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/images/white_hat_blue_red_dark.png" width="800 title="ART Red and Blue Teams">
+    <source media="(prefers-color-scheme: light)" srcset="docs/images/white_hat_blue_red.png" width="800 title="ART Red and Blue Teams">
+    <img alt="Fallback image description" src="default-image.png" width="800">
+  </picture>
+</div>
 <br />
 
 ## 学到更多

README.md

@@ -1,6 +1,6 @@
-# Adversarial Robustness Toolbox (ART) v1.19
+# Adversarial Robustness Toolbox (ART) v1.20
 <p align="center">
-  <img src="docs/images/art_lfai.png?raw=true" width="467" title="ART logo">
+  <img src="https://raw.githubusercontent.com/Trusted-AI/adversarial-robustness-toolbox/main/docs/images/art_lfai.png" width="467" title="ART logo">
 </p>
 <br />
 
@@ -18,9 +18,14 @@
 
 [中文README请按此处](README-cn.md)
 
-<p align="center">
-  <img src="https://raw.githubusercontent.com/lfai/artwork/master/lfaidata-assets/lfaidata-project-badge/graduate/color/lfaidata-project-badge-graduate-color.png" alt="LF AI & Data" width="300"/>
-</p>
+ <div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/images/lfaidata-project-badge-graduate-color_dark.png" width="400" title="LF AI & Data">
+    <source media="(prefers-color-scheme: light)" srcset="docs/images/lfaidata-project-badge-graduate-color.png" width="400" title="LF AI & Data">
+    <img alt="Fallback image description" src="default-image.png" width="400">
+  </picture>
+</div>
+<br />
 
 Adversarial Robustness Toolbox (ART) is a Python library for Machine Learning Security. ART is hosted by the 
 [Linux Foundation AI & Data Foundation](https://lfaidata.foundation) (LF AI & Data). ART provides tools that enable
@@ -32,17 +37,28 @@ generation, certification, etc.).
 
 ## Adversarial Threats
 
+ <div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/images/adversarial_threats_attacker_dark.png" width="400 title="ART Threats">
+    <source media="(prefers-color-scheme: light)" srcset="docs/images/adversarial_threats_attacker.png" width="400 title="ART Threats">
+    <img alt="Fallback image description" src="default-image.png" width="400">
+  </picture>
+</div>
+
 <p align="center">
-  <img src="docs/images/adversarial_threats_attacker.png?raw=true" width="400" title="ART logo">
-  <img src="docs/images/adversarial_threats_art.png?raw=true" width="400" title="ART logo">
+  <img src="docs/images/adversarial_threats_art.png?raw=true" width="400" title="ART Matrix">
 </p>
 <br />
 
 ## ART for Red and Blue Teams (selection)
 
-<p align="center">
-  <img src="docs/images/white_hat_blue_red.png?raw=true" width="800" title="ART Red and Blue Teams">
-</p>
+ <div align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="docs/images/white_hat_blue_red_dark.png" width="800 title="ART Red and Blue Teams">
+    <source media="(prefers-color-scheme: light)" srcset="docs/images/white_hat_blue_red.png" width="800 title="ART Red and Blue Teams">
+    <img alt="Fallback image description" src="default-image.png" width="800">
+  </picture>
+</div>
 <br />
 
 ## Learn more

art/__init__.py

@@ -13,7 +13,7 @@
 from art import preprocessing
 
 # Semantic Version
-__version__ = "1.19.2"
+__version__ = "1.20.1"
 
 LOGGING = {
     "version": 1,

art/estimators/certification/derandomized_smoothing/pytorch.py

@@ -406,12 +406,15 @@ def get_models(cls, generate_from_null: bool = False) -> list[str]:
         return supported
 
     @staticmethod
-    def create_vision_transformer(variant: str, pretrained: bool = False, **kwargs) -> "PyTorchVisionTransformer":
+    def create_vision_transformer(
+        variant: str, pretrained: bool = False, use_naflex: bool | None = None, **kwargs
+    ) -> "PyTorchVisionTransformer":
         """
         Creates a vision transformer using PyTorchViT which controls the forward pass of the model
 
         :param variant: The name of the vision transformer to load
         :param pretrained: If to load pre-trained weights
+        :param use_naflex:  If using NaFlexVit.
         :return: A ViT with the required methods needed for ART
         """
 

art/estimators/certification/derandomized_smoothing/vision_transformers/pytorch.py

@@ -167,11 +167,12 @@ def drop_tokens(x: torch.Tensor, indexes: torch.Tensor) -> torch.Tensor:
         x_no_cl = torch.reshape(x_no_cl, shape=(shape[0], -1, shape[-1]))
         return torch.cat((cls_token, x_no_cl), dim=1)
 
-    def forward_features(self, x: torch.Tensor) -> torch.Tensor:
+    def forward_features(self, x: torch.Tensor, attn_mask: torch.Tensor | None = None) -> torch.Tensor:
         """
         The forward pass of the ViT.
 
         :param x: Input data.
+        :param attn_mask: Attention mask.
         :return: The input processed by the ViT backbone
         """
 

art/estimators/object_detection/pytorch_object_detector.py

@@ -66,7 +66,6 @@ def __init__(
             "loss_rpn_box_reg",
         ),
         device_type: str = "gpu",
-        is_yolov8: bool = False,
     ):
         """
         Initialization.
@@ -94,7 +93,6 @@ def __init__(
                               'loss_objectness', and 'loss_rpn_box_reg'.
         :param device_type: Type of device to be used for model and tensors, if `cpu` run on CPU, if `gpu` run on GPU
                             if available otherwise run on CPU.
-        :param is_yolov8: The flag to be used for marking the YOLOv8 model.
         """
         import torch
         import torchvision
@@ -139,11 +137,7 @@ def __init__(
 
         self._model: torch.nn.Module
         self._model.to(self._device)
-        self.is_yolov8 = is_yolov8
-        if self.is_yolov8:
-            self._model.model.eval()  # type: ignore
-        else:
-            self._model.eval()
+        self._model.eval()
 
     @property
     def native_label_is_pytorch_format(self) -> bool:
@@ -412,10 +406,7 @@ def predict(self, x: np.ndarray, batch_size: int = 128, **kwargs) -> list[dict[s
         from torch.utils.data import TensorDataset, DataLoader
 
         # Set model to evaluation mode
-        if self.is_yolov8:
-            self._model.model.eval()  # type: ignore
-        else:
-            self._model.eval()
+        self._model.eval()
 
         # Apply preprocessing and convert to tensors
         x_preprocessed, _ = self._preprocess_and_convert_inputs(x=x, y=None, fit=False, no_grad=True)

art/estimators/object_detection/pytorch_yolo.py

@@ -16,7 +16,7 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 """
-This module implements the task specific estimator for PyTorch YOLO v3 and v5 object detectors.
+This module implements the task specific estimator for PyTorch YOLO v3, v5, v8+ object detectors.
 
 | Paper link: https://arxiv.org/abs/1804.02767
 """
@@ -42,7 +42,7 @@
 
 class PyTorchYolo(PyTorchObjectDetector):
     """
-    This module implements the model- and task specific estimator for YOLO v3, v5 object detector models in PyTorch.
+    This module implements the model- and task specific estimator for YOLO object detector models in PyTorch.
 
     | Paper link: https://arxiv.org/abs/1804.02767
     """
@@ -64,12 +64,13 @@ def __init__(
             "loss_rpn_box_reg",
         ),
         device_type: str = "gpu",
-        is_yolov8: bool = False,
+        is_ultralytics: bool = False,
+        model_name: str | None = None,
     ):
         """
         Initialization.
 
-        :param model: YOLO v3 or v5 model wrapped as demonstrated in examples/get_started_yolo.py.
+        :param model: YOLO v3, v5, or v8+ model wrapped as demonstrated in examples/get_started_yolo.py.
                       The output of the model is `list[dict[str, torch.Tensor]]`, one for each input image.
                       The fields of the dict are as follows:
 
@@ -93,8 +94,13 @@ def __init__(
                               'loss_objectness', and 'loss_rpn_box_reg'.
         :param device_type: Type of device to be used for model and tensors, if `cpu` run on CPU, if `gpu` run on GPU
                             if available otherwise run on CPU.
-        :param is_yolov8: The flag to be used for marking the YOLOv8 model.
+        :param model_name: The name of the model (e.g., 'yolov8n', 'yolov10n') for determining loss function.
         """
+        if is_ultralytics:
+            from art.estimators.object_detection.pytorch_yolo_loss_wrapper import PyTorchYoloLossWrapper
+
+            model = PyTorchYoloLossWrapper(model, model_name)
+
         super().__init__(
             model=model,
             input_shape=input_shape,
@@ -106,7 +112,6 @@ def __init__(
             preprocessing=preprocessing,
             attack_losses=attack_losses,
             device_type=device_type,
-            is_yolov8=is_yolov8,
         )
 
     def _translate_labels(self, labels: list[dict[str, "torch.Tensor"]]) -> "torch.Tensor":
@@ -154,20 +159,31 @@ def _translate_predictions(self, predictions: "torch.Tensor") -> list[dict[str,
         Translate object detection predictions from the model format (YOLO) to ART format (torchvision) and
         convert tensors to numpy arrays.
 
-        :param predictions: Object detection labels in format xcycwh (YOLO).
+        :param predictions: Object detection labels in format xcycwh (YOLO) or list of dicts (YOLO v8+).
         :return: Object detection labels in format x1y1x2y2 (torchvision).
         """
         import torch
 
+        predictions_x1y1x2y2: list[dict[str, np.ndarray]] = []
+
+        # Handle YOLO v8+ predictions (list of dicts)
+        if isinstance(predictions, list) and len(predictions) > 0 and isinstance(predictions[0], dict):
+            for pred in predictions:
+                prediction = {}
+                prediction["boxes"] = pred["boxes"].detach().cpu().numpy()
+                prediction["labels"] = pred["labels"].detach().cpu().numpy()
+                prediction["scores"] = pred["scores"].detach().cpu().numpy()
+                predictions_x1y1x2y2.append(prediction)
+            return predictions_x1y1x2y2
+
+        # Handle traditional YOLO predictions (tensor format)
         if self.channels_first:
             height = self.input_shape[1]
             width = self.input_shape[2]
         else:
             height = self.input_shape[0]
             width = self.input_shape[1]
 
-        predictions_x1y1x2y2: list[dict[str, np.ndarray]] = []
-
         for pred in predictions:
             boxes = torch.vstack(
                 [

art/estimators/object_detection/pytorch_yolo_loss_wrapper.py

@@ -0,0 +1,70 @@
+# MIT License
+#
+# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2025
+#
+# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
+# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
+# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+# persons to whom the Software is furnished to do so, subject to the following conditions:
+#
+# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
+# Software.
+#
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+"""
+PyTorch-specific YOLO loss wrapper for ART for yolo versions 8 and above.
+"""
+
+import torch
+
+
+class PyTorchYoloLossWrapper(torch.nn.Module):
+    """Wrapper for YOLO v8+ models to handle loss dict format."""
+
+    def __init__(self, model, name):
+        super().__init__()
+        self.model = model
+        try:
+            from ultralytics.models.yolo.detect import DetectionPredictor
+            from ultralytics.utils.loss import v8DetectionLoss, E2EDetectLoss
+
+            self.detection_predictor = DetectionPredictor()
+            self.model.args = self.detection_predictor.args
+            if "v10" in name:
+                self.model.criterion = E2EDetectLoss(model)
+            else:
+                self.model.criterion = v8DetectionLoss(model)
+        except ImportError as e:
+            raise ImportError("The 'ultralytics' package is required for YOLO v8+ models but not installed.") from e
+
+    def forward(self, x, targets=None):
+        """Transforms the target to dict expected by model.loss"""
+        if self.training:
+            if targets is None:
+                raise ValueError("Targets should not be None when training.")
+            items = {}
+            items["batch_idx"] = targets[:, 0]
+            items["bboxes"] = targets[:, 2:6]
+            items["cls"] = targets[:, 1]
+            items["img"] = x
+            loss, loss_components = self.model.loss(items)
+            loss_components_dict = {"loss_total": loss.sum()}
+            loss_components_dict["loss_box"] = loss_components[0].sum()
+            loss_components_dict["loss_cls"] = loss_components[1].sum()
+            loss_components_dict["loss_dfl"] = loss_components[2].sum()
+            return loss_components_dict
+        else:
+            preds = self.model(x)
+            self.detection_predictor.model = self.model
+            self.detection_predictor.batch = [x]
+            preds = self.detection_predictor.postprocess(preds, x, x)
+            items = []
+            for pred in preds:
+                items.append(
+                    {"boxes": pred.boxes.xyxy, "scores": pred.boxes.conf, "labels": pred.boxes.cls.type(torch.int)}
+                )
+            return items