Merge pull request #2662 from Trusted-AI/dev_1.19.2

Update to ART 1.19.2

Author: beat-buesser

.github/actions/tfv2-faster-rcnn/Dockerfile

@@ -1,6 +1,8 @@
 # Get base from a tensorflow image
 FROM tensorflow/tensorflow:2.13.0
 
+ENV MONO_TLS_PROVIDER=legacy
+
 # Set to install things in non-interactive mode
 ENV DEBIAN_FRONTEND noninteractive
 
@@ -22,15 +24,6 @@ RUN apt-get update \
      && apt-get clean all \
      && rm -r /var/lib/apt/lists/*
 
-RUN wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-RUN bash Miniconda3-latest-Linux-x86_64.sh -b -p /miniconda
-RUN /miniconda/bin/conda install --yes \
-    astropy \
-    matplotlib \
-    pandas \
-    scikit-learn \
-    scikit-image
-
 # Install necessary libraries for tensorflow faster rcnn
 RUN mkdir TensorFlow
 RUN cd TensorFlow && git clone https://github.com/tensorflow/models
@@ -40,7 +33,4 @@ RUN cd TensorFlow/models/research && /TensorFlow/protobuf/bin/protoc object_dete
 RUN cd TensorFlow/models/research && cp object_detection/packages/tf2/setup.py .
 RUN cd TensorFlow/models/research && python -m pip install .
 
-RUN pip install tqdm
-RUN pip install requests
-RUN pip install numba==0.50.0
-RUN pip install pytest-cov
+RUN pip install tqdm requests numba==0.50 pytest-cov numpy==1.23.5

art/attacks/evasion/auto_conjugate_gradient.py

@@ -637,8 +637,8 @@ def _check_params(self) -> None:
         if not isinstance(self.batch_size, int) or self.batch_size <= 0:
             raise ValueError("The argument batch_size has to be of type int and larger than zero.")
 
-        # if self.loss_type not in self._predefined_losses:
-        #     raise ValueError("The argument loss_type has to be either {}.".format(self._predefined_losses))
+        if self.loss_type not in self._predefined_losses:
+            raise ValueError("The argument loss_type has to be either {}.".format(self._predefined_losses))
 
         if not isinstance(self.verbose, bool):
             raise ValueError("The argument `verbose` has to be of type bool.")

art/attacks/evasion/graphite/utils.py

@@ -56,8 +56,6 @@
     from art.utils import CLASSIFIER_NEURALNETWORK_TYPE
     import torch
 
-_estimator_requirements = (BaseEstimator, ClassifierMixin)
-
 
 def dist2pixels(dist: float, width: float, obj_width: float = 30) -> float:
     """

art/attacks/evasion/overload/box_iou.py

@@ -93,10 +93,15 @@ def __init__(
                             if available otherwise run on CPU.
         """
         import torchvision
+        from torchvision.models import ResNet50_Weights
+        from torchvision.models.detection import FasterRCNN_ResNet50_FPN_Weights
 
         if model is None:  # pragma: no cover
             model = torchvision.models.detection.fasterrcnn_resnet50_fpn(
-                pretrained=True, progress=True, num_classes=91, pretrained_backbone=True
+                weights=FasterRCNN_ResNet50_FPN_Weights.COCO_V1,
+                progress=True,
+                num_classes=91,
+                weights_backbone=ResNet50_Weights.IMAGENET1K_V1,
             )
 
         super().__init__(

art/estimators/object_detection/pytorch_faster_rcnn.py

@@ -310,6 +310,7 @@ def _get_losses(
         self._model.train()
 
         self.set_dropout(False)
+        self.set_batchnorm(False)
         self.set_multihead_attention(False)
 
         # Apply preprocessing and convert to tensors

art/estimators/object_detection/pytorch_object_detector.py

@@ -1575,6 +1575,9 @@ def _extract(full_path: str, path: str) -> bool:
         return False
 
     try:
+        for entry in archive.getmembers():  # type: ignore
+            if os.path.isabs(entry.name) or ".." in entry.name:
+                raise ValueError(f"Illegal tar archive entry: {entry.name}")
         archive.extractall(path)
     except (tarfile.TarError, RuntimeError, KeyboardInterrupt):  # pragma: no cover
         if os.path.exists(path):

art/utils.py

@@ -118,14 +118,17 @@ def extract_predictions(predictions_):
 
     predictions_boxes = predictions_boxes[: predictions_t + 1]
     predictions_class = predictions_class[: predictions_t + 1]
+    predictions_scores = predictions_score[: predictions_t + 1]
 
-    return predictions_class, predictions_boxes, predictions_class
+    return predictions_class, predictions_boxes, predictions_scores
 
 
 def plot_image_with_boxes(img, boxes, pred_cls):
-    text_size = 5
-    text_th = 5
-    rect_th = 6
+    text_size = 2
+    text_th = 2
+    rect_th = 2
+
+    img = img.copy()
 
     for i in range(len(boxes)):
         # Draw Rectangle with the coordinates
@@ -155,23 +158,22 @@ def plot_image_with_boxes(img, boxes, pred_cls):
 def main():
     # Create ART object detector
     frcnn = PyTorchFasterRCNN(
-        clip_values=(0, 255), attack_losses=["loss_classifier", "loss_box_reg", "loss_objectness", "loss_rpn_box_reg"]
+        clip_values=(0, 255),
+        channels_first=True,
+        attack_losses=["loss_classifier", "loss_box_reg", "loss_objectness", "loss_rpn_box_reg"],
     )
 
     # Load image 1
     image_0 = cv2.imread("./10best-cars-group-cropped-1542126037.jpg")
     image_0 = cv2.cvtColor(image_0, cv2.COLOR_BGR2RGB)  # Convert to RGB
-    print("image_0.shape:", image_0.shape)
 
     # Load image 2
     image_1 = cv2.imread("./banner-diverse-group-of-people-2.jpg")
     image_1 = cv2.cvtColor(image_1, cv2.COLOR_BGR2RGB)  # Convert to RGB
     image_1 = cv2.resize(image_1, dsize=(image_0.shape[1], image_0.shape[0]), interpolation=cv2.INTER_CUBIC)
-    print("image_1.shape:", image_1.shape)
 
     # Stack images
     image = np.stack([image_0, image_1], axis=0).astype(np.float32)
-    print("image.shape:", image.shape)
 
     for i in range(image.shape[0]):
         plt.axis("off")
@@ -180,21 +182,23 @@ def main():
         plt.show()
 
     # Make prediction on benign samples
-    predictions = frcnn.predict(x=image)
+    image_chw = np.transpose(image, (0, 3, 1, 2))
+    predictions = frcnn.predict(x=image_chw)
 
     for i in range(image.shape[0]):
         print("\nPredictions image {}:".format(i))
 
         # Process predictions
-        predictions_class, predictions_boxes, predictions_class = extract_predictions(predictions[i])
+        predictions_class, predictions_boxes, _ = extract_predictions(predictions[i])
 
         # Plot predictions
-        plot_image_with_boxes(img=image[i].copy(), boxes=predictions_boxes, pred_cls=predictions_class)
+        plot_image_with_boxes(img=image[i], boxes=predictions_boxes, pred_cls=predictions_class)
 
     # Create and run attack
     eps = 32
-    attack = ProjectedGradientDescent(estimator=frcnn, eps=eps, eps_step=2, max_iter=10)
-    image_adv = attack.generate(x=image, y=None)
+    attack = ProjectedGradientDescent(estimator=frcnn, eps=eps, eps_step=2, max_iter=2)
+    image_adv_chw = attack.generate(x=image_chw, y=None)
+    image_adv = np.transpose(image_adv_chw, (0, 2, 3, 1))
 
     print("\nThe attack budget eps is {}".format(eps))
     print("The resulting maximal difference in pixel values is {}.".format(np.amax(np.abs(image - image_adv))))
@@ -205,16 +209,16 @@ def main():
         plt.imshow(image_adv[i].astype(np.uint8), interpolation="nearest")
         plt.show()
 
-    predictions_adv = frcnn.predict(x=image_adv)
+    predictions_adv = frcnn.predict(x=image_adv_chw)
 
     for i in range(image.shape[0]):
         print("\nPredictions adversarial image {}:".format(i))
 
         # Process predictions
-        predictions_adv_class, predictions_adv_boxes, predictions_adv_class = extract_predictions(predictions_adv[i])
+        predictions_adv_class, predictions_adv_boxes, _ = extract_predictions(predictions_adv[i])
 
         # Plot predictions
-        plot_image_with_boxes(img=image_adv[i].copy(), boxes=predictions_adv_boxes, pred_cls=predictions_adv_class)
+        plot_image_with_boxes(img=image_adv[i], boxes=predictions_adv_boxes, pred_cls=predictions_adv_class)
 
 
 if __name__ == "__main__":

examples/application_object_detection.py

@@ -282,7 +282,7 @@ def append_loss_history(loss_history, output):
 
     for i in range(config["max_iter"]):
         print("Iteration:", i)
-        patch = attack.generate(x)
+        _ = attack.generate(x)
         x_patch = attack.apply_patch(x)
 
         loss = get_loss(frcnn, x_patch, y)

examples/get_started_fasterrcnn.py

@@ -115,37 +115,45 @@
 ]
 
 
-def extract_predictions(predictions_, conf_thresh):
+def extract_predictions(predictions_, top_k):
     # Get the predicted class
     predictions_class = [COCO_INSTANCE_CATEGORY_NAMES[i] for i in list(predictions_["labels"])]
-    #  print("\npredicted classes:", predictions_class)
-    if len(predictions_class) < 1:
-        return [], [], []
+
     # Get the predicted bounding boxes
     predictions_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(predictions_["boxes"])]
 
     # Get the predicted prediction score
     predictions_score = list(predictions_["scores"])
-    # print("predicted score:", predictions_score)
+
+    # sort all lists according to scores
+    # Combine into a list of tuples
+    combined = list(zip(predictions_score, predictions_boxes, predictions_class))
+
+    # Sort by score (first element of tuple), descending
+    combined_sorted = sorted(combined, key=lambda x: x[0], reverse=True)
+
+    # Unpack sorted tuples
+    predictions_score, predictions_boxes, predictions_class = zip(*combined_sorted)
+
+    # Convert back to lists
+    predictions_score = list(predictions_score)
+    predictions_boxes = list(predictions_boxes)
+    predictions_class = list(predictions_class)  # Combine into a list of tuples
 
     # Get a list of index with score greater than threshold
-    threshold = conf_thresh
-    predictions_t = [predictions_score.index(x) for x in predictions_score if x > threshold]
-    if len(predictions_t) == 0:
-        return [], [], []
-
-    # predictions in score order
-    predictions_boxes = [predictions_boxes[i] for i in predictions_t]
-    predictions_class = [predictions_class[i] for i in predictions_t]
-    predictions_scores = [predictions_score[i] for i in predictions_t]
+    predictions_t = top_k
+
+    predictions_boxes = predictions_boxes[:predictions_t]
+    predictions_class = predictions_class[:predictions_t]
+    predictions_scores = predictions_score[:predictions_t]
+
     return predictions_class, predictions_boxes, predictions_scores
 
 
 def plot_image_with_boxes(img, boxes, pred_cls, title):
-    plt.style.use("ggplot")
-    text_size = 1
-    text_th = 3
-    rect_th = 1
+    text_size = 2
+    text_th = 2
+    rect_th = 2
 
     img = img.copy()
 
@@ -175,18 +183,10 @@ def plot_image_with_boxes(img, boxes, pred_cls, title):
     plt.show()
 
 
-"""
-#################        Evasion settings        #################
-"""
-eps = 32
-eps_step = 2
-max_iter = 10
-
-
 """
 #################        Model definition        #################
 """
-MODEL = "yolov3"  # OR yolov5
+MODEL = "yolov5"  # OR yolov5
 
 
 if MODEL == "yolov3":
@@ -265,35 +265,37 @@ def forward(self, x, targets=None):
 """
 response = requests.get("https://ultralytics.com/images/zidane.jpg")
 img = np.asarray(Image.open(BytesIO(response.content)).resize((640, 640)))
-img_reshape = img.transpose((2, 0, 1))
-image = np.stack([img_reshape], axis=0).astype(np.float32)
-x = image.copy()
+image = np.stack([img], axis=0).astype(np.float32)
+image_chw = np.transpose(image, (0, 3, 1, 2))
 
 """
 #################        Evasion attack        #################
 """
 
-attack = ProjectedGradientDescent(estimator=detector, eps=eps, eps_step=eps_step, max_iter=max_iter)
-image_adv = attack.generate(x=x, y=None)
+eps = 32
+attack = ProjectedGradientDescent(estimator=detector, eps=eps, eps_step=2, max_iter=10)
+image_adv_chw = attack.generate(x=image_chw, y=None)
+image_adv = np.transpose(image_adv_chw, (0, 2, 3, 1))
 
 print("\nThe attack budget eps is {}".format(eps))
-print("The resulting maximal difference in pixel values is {}.".format(np.amax(np.abs(x - image_adv))))
+print("The resulting maximal difference in pixel values is {}.".format(np.amax(np.abs(image_chw - image_adv_chw))))
 
 plt.axis("off")
 plt.title("adversarial image")
-plt.imshow(image_adv[0].transpose(1, 2, 0).astype(np.uint8), interpolation="nearest")
+plt.imshow(image_adv[0].astype(np.uint8), interpolation="nearest")
 plt.show()
 
-threshold = 0.85  # 0.5
-dets = detector.predict(x)
-preds = extract_predictions(dets[0], threshold)
-plot_image_with_boxes(img=img, boxes=preds[1], pred_cls=preds[0], title="Predictions on original image")
+predictions = detector.predict(x=image_chw)
+predictions_class, predictions_boxes, _ = extract_predictions(predictions[0], top_k=3)
+plot_image_with_boxes(
+    img=image[0], boxes=predictions_boxes, pred_cls=predictions_class, title="Predictions on original image"
+)
 
-dets = detector.predict(image_adv)
-preds = extract_predictions(dets[0], threshold)
+predictions = detector.predict(image_adv_chw)
+predictions_class, predictions_boxes, d = extract_predictions(predictions[0], top_k=3)
 plot_image_with_boxes(
-    img=image_adv[0].transpose(1, 2, 0).copy(),
-    boxes=preds[1],
-    pred_cls=preds[0],
+    img=image_adv[0],
+    boxes=predictions_boxes,
+    pred_cls=predictions_class,
     title="Predictions on adversarial image",
 )