Extracting gradient class activation map (grad CAM)

[ljtruong]

Hi,

I was wondering if anyone had a guide on how to extract the faster rcnn network (faster_rcnn_R_50_FPN_3x.yaml) features for input into grad CAM. Using the partial execution method.

The aim is to extract proposed roi crops showing the activation maps for each detected class in an image at the fc layer (or any layer)

I'ved use this as my start code to extract the features from the predicted proposal regions. but wasn't sure how to input them into grad cam.
https://stackoverflow.com/a/62724033

I've gotten it to a reasonable point with a hacky way

1. load the image

# open the image and preprocess 
# simple preprocess for now - Resize
image = cv2.imread('notebooks/000000001296.jpg')
image_org = cv2.imread('notebooks/000000001296.jpg')
height, width = image.shape[:2]

height, width = 800, 1280
image = cv2.resize(image, (width,height))
image_org_resized = cv2.resize(image_org, (width,height))
# resize image
image = torch.as_tensor(np.ascontiguousarray(image.astype('float32').transpose(2, 0, 1))).requires_grad_(True)

2. load the model

# load the model
images = ImageList.from_tensors([image], size_divisibility=32).to("cuda")  # preprocessed input tensor
#setup config
cfg = get_cfg()
cfg.merge_from_file(model_zoo.get_config_file("COCO-detection/faster_rcnn_R_50_FPN_3x.yaml"))
cfg.MODEL.WEIGHTS = os.path.join("notebooks/models/model_final_280758.pkl")
cfg.SOLVER.IMS_PER_BATCH = 1
cfg.DATALOADER.NUM_WORKERS = 0
#cfg.MODEL.ROI_HEADS.NUM_CLASSES = 80  
#Just run these lines if you have the trained model im memory
cfg.MODEL.ROI_HEADS.SCORE_THRESH_TEST = 0.7   # set the testing threshold for this model
#build model
model = build_model(cfg)
DetectionCheckpointer(model).load("notebooks/models/model_final_280758.pkl")
model.eval()#make sure its in eval mode

3. feed the inputs to the model

# pass the model through the network
inputs = [{"image": image, "height": height, "width": width}]
with torch.no_grad():
    images = model.preprocess_image(inputs)  # don't forget to preprocess
    features = model.backbone(images.tensor)  # set of cnn features
    res_features = model.backbone.bottom_up(images.tensor)
    proposals, _ = model.proposal_generator(images, features, None)  # RPN

    features_ = [features[f] for f in model.roi_heads.box_in_features]
    box_features_pooler = model.roi_heads.box_pooler(features_, [x.proposal_boxes for x in proposals])
    box_features = model.roi_heads.box_head(box_features_pooler)  # features of all 1k candidates
    predictions = model.roi_heads.box_predictor(box_features) # applies box deltas to readjust proposal boxes, computes non-maximum supression, score threshold and ranks top-k
    #losses = predictions[1]
    
    pred_instances, pred_inds = model.roi_heads.box_predictor.inference(predictions, proposals)
    pred_instances = model.roi_heads.forward_with_given_boxes(features, pred_instances)

    # output boxes, masks, scores, etc
    pred_instances_out = model._postprocess(pred_instances, inputs, images.image_sizes)  # scale box to orig size
    # features of the proposed boxes
    feats = box_features[pred_inds]

4. using weights from the partial execution retrieve the activated weights
   I've worked with the weights from res4 as it was simpler to work with. Looking to get res5 for stronger activation

def getCAM(feature_conv, weight_fc, class_idx):
    # feature_conv.shape = (1, 2048, 7, 7)
    _, nc, h, w = feature_conv.shape
    # reshaped to (2048, 49)
    cam = weight_fc[class_idx].dot(feature_conv.reshape((nc, h*w))) 
    cam = cam.reshape(h, w)
    cam = cam - np.min(cam)
    cam_img = cam / np.max(cam)
    return [cam_img]

feature_conv = res_features['res4'].cpu().data.numpy() # retrieve the feature weights from the layer you are looking for
weight_fc_params = box_features # take the features from the fully connected layer
weight_fc = np.squeeze(weight_fc_params.cpu().data.numpy())

class_idx = 0 # TODO how to select the right row.
overlay = getCAM(feature_conv, weight_fc, 0)

plt.imshow(overlay[0], alpha=0.5, cmap='jet')
plt.show()

cam_resized = cv2.resize(overlay[0], (width,height))
plt.imshow(image_org_resized)
plt.imshow(cam_resized, alpha=0.5, cmap='jet')
plt.show()

output of current attempt
https://imgur.com/a/qBgMAq8

---

This is a snippet of what I'm looking to achieve

from PIL import Image
from matplotlib.pyplot import imshow
from torchvision import models, transforms
from torch.autograd import Variable
from torch.nn import functional as F
from torch import topk
import numpy as np
import skimage.transform

image = Image.open("cat.jpg")
#plt.imshow(image)

# Imagenet mean/std

normalize = transforms.Normalize(
   mean=[0.485, 0.456, 0.406],
   std=[0.229, 0.224, 0.225]
)

# Preprocessing - scale to 224x224 for model, convert to tensor, 
# and normalize to -1..1 with mean/std for ImageNet

preprocess = transforms.Compose([
   transforms.Resize((224,224)),
   transforms.ToTensor(),
   normalize
])

display_transform = transforms.Compose([
   transforms.Resize((224,224))])

tensor = preprocess(image)

prediction_var = Variable((tensor.unsqueeze(0)).cuda(), requires_grad=True)

model = models.resnet50(pretrained=True)

model.cuda()
model.eval()

class SaveFeatures():
    features=None
    def __init__(self, m): 
            self.hook = m.register_forward_hook(self.hook_fn)
    def hook_fn(self, module, input, output): 
            self.features = ((output.cpu()).data).numpy()
    def remove(self): 
            self.hook.remove()

# freeze the algorithm at layer 4, skip other layers. Then run it through the fully connected layer
final_layer = model._modules.get('layer4')
activated_features = SaveFeatures(final_layer)

prediction = model(prediction_var)
pred_probabilities = F.softmax(prediction, dim=1).data.squeeze()
activated_features.remove()

# view prediction probability and class (given as a number)
# class mapping found here. https://gist.github.com/yrevar/942d3a0ac09ec9e5eb3a
print(topk(pred_probabilities,1))

def getCAM(feature_conv, weight_fc, class_idx):
    # feature_conv.shape = (1, 2048, 7, 7)
    _, nc, h, w = feature_conv.shape
    cam = weight_fc[class_idx].dot(feature_conv.reshape((nc, h*w)))
    cam = cam.reshape(h, w)
    cam = cam - np.min(cam)
    cam_img = cam / np.max(cam)
    return [cam_img]

# extract the activation from softmax 
weight_softmax_params = list(model._modules.get('fc').parameters())
# numpy squeeze to remove single dimensional entries
# eg. (1,2048) to (2048,)
weight_softmax = np.squeeze(weight_softmax_params[0].cpu().data.numpy())

#weight_softmax_params[0].data.shape

class_idx = topk(pred_probabilities,1)[1].int()

overlay = getCAM(activated_features.features, weight_softmax, class_idx )

# show overlay
plt.imshow(overlay[0], alpha=0.5, cmap='jet')
plt.show()

# Result is a heatmap
imshow(display_transform(image))
imshow(skimage.transform.resize(overlay[0], tensor.shape[1:3]), alpha=0.5, cmap='jet')
plt.show()

Result
https://imgur.com/a/HjdT42Q

[jojo0513]

change height, width = 800, 1280 to height, width = 224, 224