Merge branch 'main' into dev_1.12.2

Author: beat-buesser

.github/workflows/ci-tensorflow-v1.yml

@@ -45,9 +45,11 @@ jobs:
           sudo apt-get update
           sudo apt-get -y -q install ffmpeg libavcodec-extra
           python -m pip install --upgrade pip setuptools wheel
-          pip install -q -r <(sed '/^pandas/d;/^scipy/d' requirements_test.txt)
+          pip install -q -r <(sed '/^pandas/d;/^scipy/d;/^matplotlib/d;/^xgboost/d' requirements_test.txt)
           pip install pandas==1.3.5
           pip install scipy==1.7.2
+          pip install matplotlib==3.5.3
+          pip install xgboost==1.6.2
           pip install tensorflow==${{ matrix.tensorflow }}
           pip install keras==${{ matrix.keras }}
           pip list

.github/workflows/dockerhub.yml

@@ -24,22 +24,22 @@ jobs:
         uses: actions/checkout@v3
 
       - name: Log in to Docker Hub
-        uses: docker/login-action@49ed152c8eca782a232dede0303416e8f356c37b
+        uses: docker/login-action@f4ef78c080cd8ba55a85445d5b36e214a81df20a
         with:
           username: ${{ secrets.DOCKER_HUB_USERNAME }}
           password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}
 
       - name: Extract metadata (tags, labels) for Docker
         id: meta
-        uses: docker/metadata-action@69f6fc9d46f2f8bf0d5491e4aabe0bb8c6a4678a
+        uses: docker/metadata-action@57396166ad8aefe6098280995947635806a0e6ea
         with:
           images: adversarialrobustnesstoolbox/releases
           tags: |
             type=raw,value={{branch}}-1.12.1-{{sha}}
             type=semver,pattern={{version}}
 
       - name: Build and push Docker image
-        uses: docker/build-push-action@c84f38281176d4c9cdb1626ffafcd6b3911b5d94
+        uses: docker/build-push-action@c56af957549030174b10d6867f20e78cfd7debc5
         with:
           context: .
           push: true

art/estimators/object_detection/pytorch_yolo.py

@@ -16,7 +16,7 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 # SOFTWARE.
 """
-This module implements the task specific estimator for PyTorch YOLO v3 object detectors.
+This module implements the task specific estimator for PyTorch YOLO v3 and v5 object detectors.
 
 | Paper link: https://arxiv.org/abs/1804.02767
 """
@@ -89,10 +89,10 @@ def translate_predictions_xcycwh_to_x1y1x2y2(
 
 def translate_labels_art_to_yolov3(labels_art: List[Dict[str, "torch.Tensor"]]):
     """
-    Translate labels from ART to YOLO v3.
+    Translate labels from ART to YOLO v3 and v5.
 
     :param labels_art: Object detection labels in format ART (torchvision).
-    :return: Object detection labels in format YOLO v3.
+    :return: Object detection labels in format YOLO v3 and v5.
     """
     import torch  # lgtm [py/repeated-import]
 
@@ -115,7 +115,7 @@ def translate_labels_art_to_yolov3(labels_art: List[Dict[str, "torch.Tensor"]]):
 
 class PyTorchYolo(ObjectDetectorMixin, PyTorchEstimator):
     """
-    This module implements the model- and task specific estimator for YOLO v3 object detector models in PyTorch.
+    This module implements the model- and task specific estimator for YOLO v3, v5 object detector models in PyTorch.
 
     | Paper link: https://arxiv.org/abs/1804.02767
     """
@@ -142,7 +142,8 @@ def __init__(
         """
         Initialization.
 
-        :param model: Object detection model. The output of the model is `List[Dict[Tensor]]`, one for each input
+        :param model: Object detection model wrapped as demonstrated in examples/get_started_yolo.py.
+                      The output of the model is `List[Dict[Tensor]]`, one for each input
                       image. The fields of the Dict are as follows:
 
                       - boxes (FloatTensor[N, 4]): the predicted boxes in [x1, y1, x2, y2] format, with values \

examples/get_started_yolo.py

@@ -0,0 +1,297 @@
+"""
+The script demonstrates a simple example of using ART with YOLO (versions 3 and 5).
+The example loads a YOLO model pretrained on the COCO dataset
+and creates an adversarial example using Projected Gradient Descent method.
+
+- To use Yolov3, run:
+        pip install pytorchyolo
+
+- To use Yolov5, run:
+        pip install yolov5
+
+Note: If pytorchyolo throws an error in pytorchyolo/utils/loss.py, add before line 174 in that file, the following:
+        gain = gain.to(torch.int64)
+"""
+
+import requests
+import numpy as np
+from PIL import Image
+from io import BytesIO
+import torch
+
+from art.estimators.object_detection.pytorch_yolo import PyTorchYolo
+from art.attacks.evasion import ProjectedGradientDescent
+
+import cv2
+import matplotlib
+import matplotlib.pyplot as plt
+
+
+"""
+#################        Helper functions and labels          #################
+"""
+
+COCO_INSTANCE_CATEGORY_NAMES = [
+    "person",
+    "bicycle",
+    "car",
+    "motorcycle",
+    "airplane",
+    "bus",
+    "train",
+    "truck",
+    "boat",
+    "traffic light",
+    "fire hydrant",
+    "stop sign",
+    "parking meter",
+    "bench",
+    "bird",
+    "cat",
+    "dog",
+    "horse",
+    "sheep",
+    "cow",
+    "elephant",
+    "bear",
+    "zebra",
+    "giraffe",
+    "backpack",
+    "umbrella",
+    "handbag",
+    "tie",
+    "suitcase",
+    "frisbee",
+    "skis",
+    "snowboard",
+    "sports ball",
+    "kite",
+    "baseball bat",
+    "baseball glove",
+    "skateboard",
+    "surfboard",
+    "tennis racket",
+    "bottle",
+    "wine glass",
+    "cup",
+    "fork",
+    "knife",
+    "spoon",
+    "bowl",
+    "banana",
+    "apple",
+    "sandwich",
+    "orange",
+    "broccoli",
+    "carrot",
+    "hot dog",
+    "pizza",
+    "donut",
+    "cake",
+    "chair",
+    "couch",
+    "potted plant",
+    "bed",
+    "dining table",
+    "toilet",
+    "tv",
+    "laptop",
+    "mouse",
+    "remote",
+    "keyboard",
+    "cell phone",
+    "microwave",
+    "oven",
+    "toaster",
+    "sink",
+    "refrigerator",
+    "book",
+    "clock",
+    "vase",
+    "scissors",
+    "teddy bear",
+    "hair drier",
+    "toothbrush",
+]
+
+
+def extract_predictions(predictions_, conf_thresh):
+    # Get the predicted class
+    predictions_class = [COCO_INSTANCE_CATEGORY_NAMES[i] for i in list(predictions_["labels"])]
+    #  print("\npredicted classes:", predictions_class)
+    if len(predictions_class) < 1:
+        return [], [], []
+    # Get the predicted bounding boxes
+    predictions_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(predictions_["boxes"])]
+
+    # Get the predicted prediction score
+    predictions_score = list(predictions_["scores"])
+    # print("predicted score:", predictions_score)
+
+    # Get a list of index with score greater than threshold
+    threshold = conf_thresh
+    predictions_t = [predictions_score.index(x) for x in predictions_score if x > threshold]
+    if len(predictions_t) > 0:
+        predictions_t = predictions_t  # [-1] #indices where score over threshold
+    else:
+        # no predictions esxceeding threshold
+        return [], [], []
+    # predictions in score order
+    predictions_boxes = [predictions_boxes[i] for i in predictions_t]
+    predictions_class = [predictions_class[i] for i in predictions_t]
+    predictions_scores = [predictions_score[i] for i in predictions_t]
+    return predictions_class, predictions_boxes, predictions_scores
+
+
+def plot_image_with_boxes(img, boxes, pred_cls, title):
+    plt.style.use("ggplot")
+    text_size = 1
+    text_th = 3
+    rect_th = 1
+
+    for i in range(len(boxes)):
+        cv2.rectangle(
+            img,
+            (int(boxes[i][0][0]), int(boxes[i][0][1])),
+            (int(boxes[i][1][0]), int(boxes[i][1][1])),
+            color=(0, 255, 0),
+            thickness=rect_th,
+        )
+        # Write the prediction class
+        cv2.putText(
+            img,
+            pred_cls[i],
+            (int(boxes[i][0][0]), int(boxes[i][0][1])),
+            cv2.FONT_HERSHEY_SIMPLEX,
+            text_size,
+            (0, 255, 0),
+            thickness=text_th,
+        )
+
+    plt.figure()
+    plt.axis("off")
+    plt.title(title)
+    plt.imshow(img.astype(np.uint8), interpolation="nearest")
+    plt.show()
+
+
+"""
+#################        Evasion settings        #################
+"""
+eps = 32
+eps_step = 2
+max_iter = 10
+
+
+"""
+#################        Model definition        #################
+"""
+MODEL = "yolov3"  # OR yolov5
+
+
+if MODEL == "yolov3":
+
+    from pytorchyolo.utils.loss import compute_loss
+    from pytorchyolo.models import load_model
+
+    class Yolo(torch.nn.Module):
+        def __init__(self, model):
+            super().__init__()
+            self.model = model
+
+        def forward(self, x, targets=None):
+            if self.training:
+                outputs = self.model(x)
+                loss, loss_components = compute_loss(outputs, targets, self.model)
+                loss_components_dict = {"loss_total": loss}
+                return loss_components_dict
+            else:
+                return self.model(x)
+
+    model_path = "./yolov3.cfg"
+    weights_path = "./yolov3.weights"
+    model = load_model(model_path=model_path, weights_path=weights_path)
+
+    model = Yolo(model)
+
+    detector = PyTorchYolo(
+        model=model, device_type="cpu", input_shape=(3, 640, 640), clip_values=(0, 255), attack_losses=("loss_total",)
+    )
+
+elif MODEL == "yolov5":
+
+    import yolov5
+    from yolov5.utils.loss import ComputeLoss
+
+    matplotlib.use("TkAgg")
+
+    class Yolo(torch.nn.Module):
+        def __init__(self, model):
+            super().__init__()
+            self.model = model
+            self.model.hyp = {
+                "box": 0.05,
+                "obj": 1.0,
+                "cls": 0.5,
+                "anchor_t": 4.0,
+                "cls_pw": 1.0,
+                "obj_pw": 1.0,
+                "fl_gamma": 0.0,
+            }
+            self.compute_loss = ComputeLoss(self.model.model.model)
+
+        def forward(self, x, targets=None):
+            if self.training:
+                outputs = self.model.model.model(x)
+                loss, loss_items = self.compute_loss(outputs, targets)
+                loss_components_dict = {"loss_total": loss}
+                return loss_components_dict
+            else:
+                return self.model(x)
+
+    model = yolov5.load("yolov5s.pt")
+
+    model = Yolo(model)
+
+    detector = PyTorchYolo(
+        model=model, device_type="cpu", input_shape=(3, 640, 640), clip_values=(0, 255), attack_losses=("loss_total",)
+    )
+
+
+"""
+#################        Example image        #################
+"""
+response = requests.get("https://ultralytics.com/images/zidane.jpg")
+img = np.asarray(Image.open(BytesIO(response.content)).resize((640, 640)))
+img_reshape = img.transpose((2, 0, 1))
+image = np.stack([img_reshape], axis=0).astype(np.float32)
+x = image.copy()
+
+"""
+#################        Evasion attack        #################
+"""
+
+attack = ProjectedGradientDescent(estimator=detector, eps=eps, eps_step=eps_step, max_iter=max_iter)
+image_adv = attack.generate(x=x, y=None)
+
+print("\nThe attack budget eps is {}".format(eps))
+print("The resulting maximal difference in pixel values is {}.".format(np.amax(np.abs(x - image_adv))))
+
+plt.axis("off")
+plt.title("adversarial image")
+plt.imshow(image_adv[0].transpose(1, 2, 0).astype(np.uint8), interpolation="nearest")
+plt.show()
+
+threshold = 0.85  # 0.5
+dets = detector.predict(x)
+preds = extract_predictions(dets[0], threshold)
+plot_image_with_boxes(img=img, boxes=preds[1], pred_cls=preds[0], title="Predictions on original image")
+
+dets = detector.predict(image_adv)
+preds = extract_predictions(dets[0], threshold)
+plot_image_with_boxes(
+    img=image_adv[0].transpose(1, 2, 0).copy(),
+    boxes=preds[1],
+    pred_cls=preds[0],
+    title="Predictions on adversarial image",
+)

notebooks/README.md

@@ -60,7 +60,8 @@ show how to use ART to create feature adversaries ([Sabour et al., 2016](https:/
 [attack_adversarial_patch.ipynb](adversarial_patch/attack_adversarial_patch.ipynb) [[on nbviewer](https://nbviewer.jupyter.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/adversarial_patch/attack_adversarial_patch.ipynb)]
 shows how to use ART to create real-world adversarial patches that fool real-world object detection and classification 
 models.
-[attack_adversarial_patch_TensorFlowV2.ipynb](adversarial_patch/attack_adversarial_patch.ipynb) [[on nbviewer](https://nbviewer.jupyter.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/adversarial_patch/attack_adversarial_patch_TensorFlowV2.ipynb)]  TensorFlow v2 specific attack implementation.
+[attack_adversarial_patch_TensorFlowV2.ipynb](adversarial_patch/attack_adversarial_patch.ipynb) [[on nbviewer](https://nbviewer.jupyter.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/adversarial_patch/attack_adversarial_patch_TensorFlowV2.ipynb)]  TensorFlow v2 specific attack implementation. 
+[attack_adversarial_patch_pytorch_yolo.ipynb](adversarial_patch/attack_adversarial_patch_pytorch_yolo.ipynb) [[on nbviewer](https://nbviewer.jupyter.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/adversarial_patch/attack_adversarial_patch_pytorch_yolo.ipynb)] YOLO v3 and v5 specific attack.
 
 <p align="center">
   <img src="../utils/data/images/adversarial_patch.png?raw=true" width="200" title="adversarial_patch">