yolo dectection problem

[deligenters]

i start to a yolostart, it can work ,but when i try to amplify loss_obj only，
such as config = {"attack_losses": [ "loss_box"]} ,it does 't work,this is message:

RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

the ComputeLoss just work when only contains "loss_total", in fact "loss_total" is sum of "loss_box", "loss_obj", "loss_cls"。but i just try to amplify loss_obj 。

what i can do ?

Thank in advance.

[beat-buesser]

Hi @deligenters Would you be able to share a code example for your issue?

[deligenters]

import numpy as np
from PIL import Image
import torch
from art.estimators.object_detection.pytorch_yolo import PyTorchYolo
import cv2
import matplotlib
import matplotlib.pyplot as plt


"""
#################        Helper functions and labels          #################
"""

COCO_INSTANCE_CATEGORY_NAMES = [
    "person",
    "bicycle",
    "car",
    "motorcycle",
    "airplane",
    "bus",
    "train",
    "truck",
    "boat",
    "traffic light",
    "fire hydrant",
    "stop sign",
    "parking meter",
    "bench",
    "bird",
    "cat",
    "dog",
    "horse",
    "sheep",
    "cow",
    "elephant",
    "bear",
    "zebra",
    "giraffe",
    "backpack",
    "umbrella",
    "handbag",
    "tie",
    "suitcase",
    "frisbee",
    "skis",
    "snowboard",
    "sports ball",
    "kite",
    "baseball bat",
    "baseball glove",
    "skateboard",
    "surfboard",
    "tennis racket",
    "bottle",
    "wine glass",
    "cup",
    "fork",
    "knife",
    "spoon",
    "bowl",
    "banana",
    "apple",
    "sandwich",
    "orange",
    "broccoli",
    "carrot",
    "hot dog",
    "pizza",
    "donut",
    "cake",
    "chair",
    "couch",
    "potted plant",
    "bed",
    "dining table",
    "toilet",
    "tv",
    "laptop",
    "mouse",
    "remote",
    "keyboard",
    "cell phone",
    "microwave",
    "oven",
    "toaster",
    "sink",
    "refrigerator",
    "book",
    "clock",
    "vase",
    "scissors",
    "teddy bear",
    "hair drier",
    "toothbrush",
]
def extract_predictions(predictions_, conf_thresh):
    # Get the predicted class
    predictions_class = [COCO_INSTANCE_CATEGORY_NAMES[i] for i in list(predictions_["labels"])]
    #  print("\npredicted classes:", predictions_class)
    if len(predictions_class) < 1:
        return [], [], []
    # Get the predicted bounding boxes
    predictions_boxes = [[(i[0], i[1]), (i[2], i[3])] for i in list(predictions_["boxes"])]

    # Get the predicted prediction score
    predictions_score = list(predictions_["scores"])
    # print("predicted score:", predictions_score)

    # Get a list of index with score greater than threshold
    threshold = conf_thresh
    predictions_t = [predictions_score.index(x) for x in predictions_score if x > threshold]
    if len(predictions_t) == 0:
        return [], [], []

    # predictions in score order
    predictions_boxes = [predictions_boxes[i] for i in predictions_t]
    predictions_class = [predictions_class[i] for i in predictions_t]
    predictions_scores = [predictions_score[i] for i in predictions_t]
    return predictions_class, predictions_boxes, predictions_scores
def plot_image_with_boxes(img, boxes, pred_cls, title):
    plt.style.use("ggplot")
    text_size = 1
    text_th = 3
    rect_th = 1

    for i in range(len(boxes)):
        cv2.rectangle(
            img,
            (int(boxes[i][0][0]), int(boxes[i][0][1])),
            (int(boxes[i][1][0]), int(boxes[i][1][1])),
            color=(0, 255, 0),
            thickness=rect_th,
        )
        # Write the prediction class
        cv2.putText(
            img,
            pred_cls[i],
            (int(boxes[i][0][0]), int(boxes[i][0][1])),
            cv2.FONT_HERSHEY_SIMPLEX,
            text_size,
            (0, 255, 0),
            thickness=text_th,
        )

    plt.figure()
    plt.axis("off")
    plt.title(title)
    plt.imshow(img.astype(np.uint8), interpolation="nearest")
    plt.show()
eps = 32
eps_step = 2
max_iter = 1
import yolov5
from yolov5.utils.loss import ComputeLoss

matplotlib.use("TkAgg")
# config = {"attack_losses": ["loss_total", "loss_box", "loss_obj", "loss_cls"],}   ####################################it can work
# config = {"attack_losses": [ "loss_total", "loss_box"]}   #######################################it can work
config = {"attack_losses": [ "loss_box"]}   #############################################  #it doesn't work
class Yolo(torch.nn.Module):
    def __init__(self, model):
        super().__init__()
        self.model = model
        self.model.hyp = {
            "box": 0.05,
            "obj": 1.0,
            "cls": 0.5,
            "anchor_t": 4.0,
            "cls_pw": 1.0,
            "obj_pw": 1.0,
            "fl_gamma": 0.0,
        }
        self.compute_loss = ComputeLoss(self.model.model.model)

    def forward(self, x, targets=None):
        if self.training:
            outputs = self.model.model.model(x)
            loss, loss_items = self.compute_loss(outputs, targets)
            # loss_components_dict = {"loss_total": loss,
            #                                      "loss_box": loss_items[0],      ################Modifying this won't make any difference
            #                                      "loss_obj": loss_items[1],
            #                                      "loss_cls": loss_items[2]}
            #loss_components_dict = {
            #                        "loss_box": loss_items[0],
            #                        "loss_cls": loss_items[2] ################Modifying this won't make any difference
          # }
            # loss_components_dict = {
            #
            #     "loss_total": loss_items[0],
            #     "loss_obj": loss_items[1], ################Modifying this won't make any difference
            #     "loss_cls": loss_items[2]
            # }
            return loss_components_dict
        else:


            return self.model(x)



model = yolov5.load("yolov5s.pt",device='cpu')

model = Yolo(model)
detector = PyTorchYolo(                     
    model=model, device_type="gpu", input_shape=(3, 640, 640), clip_values=(0, 255), attack_losses=config["attack_losses"]
)

img = np.asarray(Image.open('img.png').resize((640, 640)))
img_reshape = img.transpose((2, 0, 1))
image = np.stack([img_reshape], axis=0).astype(np.float32)
x = image.copy()
                                                                                
"""
#################        Evasion attack        #################
"""
from art.attacks.evasion import FastGradientMethod,AutoConjugateGradient,AutoAttack,DeepFool,ElasticNet,GeoDA,SquareAttack,ZooAttack,AutoProjectedGradientDescent,SaliencyMapMethod,FrameSaliencyAttack,BasicIterativeMethod
    # attack = ProjectedGradientDescent(estimator=detector, eps=eps, eps_step=eps_step, max_iter=max_iter)
attack = FastGradientMethod(estimator=detector, eps=5.0)
image_adv = attack.generate(x=x, y=None)

print("\nThe attack budget eps is {}".format(eps))
print("The resulting maximal difference in pixel values is {}.".format(np.amax(np.abs(x - image_adv))))

plt.axis("off")
plt.title("adversarial image")
plt.imshow(image_adv[0].transpose(1, 2, 0).astype(np.uint8), interpolation="nearest")
plt.show()

threshold = 0.5 # 0.5
dets = detector.predict(x)
preds = extract_predictions(dets[0], threshold)
plot_image_with_boxes(img=img, boxes=preds[1], pred_cls=preds[0], title="Predictions on original image")

dets = detector.predict(image_adv)
preds = extract_predictions(dets[0], threshold)
plot_image_with_boxes(
    img=image_adv[0].transpose(1, 2, 0).copy(),
    boxes=preds[1],
    pred_cls=preds[0],
    title="Predictions on adversarial image",
)

Here is the error message
``
Traceback (most recent call last):
File "/media/lumos/da/xuex/PycharmProjects/pythonProject/adversarial-robustness-toolbox-main/examples/get_started_yolo2.py", line 229, in
image_adv = attack.generate(x=x, y=None)
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/fast_gradient.py", line 316, in generate
adv_x_best = self._compute(
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/fast_gradient.py", line 540, in _compute
perturbation = self._compute_perturbation(batch, batch_labels, mask_batch, decay, momentum)
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/fast_gradient.py", line 395, in _compute_perturbation
grad = self.estimator.loss_gradient(x, y) * (1 - 2 * int(self.targeted))
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/estimators/object_detection/pytorch_yolo.py", line 426, in loss_gradient
loss.backward(retain_graph=True) # type: ignore
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
torch.autograd.backward(
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/torch/autograd/init.py", line 200, in backward
Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

Process finished with exit code 1
``

[beat-buesser]

@deligenters That's interesting. Does the error also occur if you exchange FastGradientMethod with ProjectedGradientDescent?

[deligenters]

use ProjectedGradientDescent will cause the same problem,the ComputeLoss just work when only contains "loss_total",
Here is the error message:
``
PGD - Iterations: 0%| | 0/1 [00:00<?, ?it/s]
Traceback (most recent call last):
File "/media/lumos/da/xuex/PycharmProjects/pythonProject/adversarial-robustness-toolbox-main/examples/get_started_yolo2.py", line 216, in
image_adv = attack.generate(x=x, y=None)
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/projected_gradient_descent/projected_gradient_descent.py", line 200, in generate
return self._attack.generate(x=x, y=y, **kwargs)
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_numpy.py", line 422, in generate
adv_x = self._compute(
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/fast_gradient.py", line 540, in _compute
perturbation = self._compute_perturbation(batch, batch_labels, mask_batch, decay, momentum)
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/attacks/evasion/fast_gradient.py", line 395, in _compute_perturbation
grad = self.estimator.loss_gradient(x, y) * (1 - 2 * int(self.targeted))
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/art/estimators/object_detection/pytorch_yolo.py", line 426, in loss_gradient
loss.backward(retain_graph=True) # type: ignore
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/torch/_tensor.py", line 487, in backward
torch.autograd.backward(
File "/home/lumos/anaconda3/envs/torch/lib/python3.9/site-packages/torch/autograd/init.py", line 200, in backward
Variable._execution_engine.run_backward( # Calls into the C++ engine to run the backward pass
RuntimeError: element 0 of tensors does not require grad and does not have a grad_fn

Process finished with exit code 1
``

[beat-buesser]

I think I remember the reason. It is not caused by code in ART, but a call to .detach() the following line of yolov5 defining the loss components stored loss_items:

https://github.com/ultralytics/yolov5/blob/493981cd3b87112fe0e32fe20478b83b3ef33659/utils/loss.py#L175

Therefore no gradients can flow back to the loss components. You need to manually remove the call to .detach() in your local installation of yolov5.