feature activation method

Author: jacobgil

README.md

@@ -45,7 +45,8 @@ The aim is also to serve as a benchmark of algorithms and metrics for research o
 | LayerCAM            | Spatially weight the activations by positive gradients. Works better especially in lower layers                             |
 | FullGrad            | Computes the gradients of the biases from all over the network, and then sums them                                          |
 | Deep Feature Factorizations           | Non Negative Matrix Factorization on the 2D activations                                                   |
-|  KPCA-CAM           | Like EigenCAM but with Kernel PCA instead of PCA
+| KPCA-CAM            | Like EigenCAM but with Kernel PCA instead of PCA                                                                            |            
+| FEM                 | A gradient free method that binarizes activations by an activation > mean + k * std rule.                                   |
 ## Visual Examples
 
 | What makes the network think the image label is 'pug, pug-dog' | What makes the network think the image label is 'tabby, tabby cat' | Combining Grad-CAM with Guided Backpropagation for the 'pug, pug-dog' class |
@@ -357,3 +358,8 @@ Edo Collins, Radhakrishna Achanta, Sabine Süsstrunk`
 https://arxiv.org/abs/2410.00267 <br>
 `KPCA-CAM: Visual Explainability of Deep Computer Vision Models using Kernel PCA
 Sachin Karmani, Thanushon Sivakaran, Gaurav Prasad, Mehmet Ali, Wenbo Yang, Sheyang Tang`
+
+https://hal.science/hal-02963298/document <br>
+`Features Understanding in 3D CNNs for Actions Recognition in Video
+Kazi Ahmed Asif Fuad, Pierre-Etienne Martin, Romain Giot, Romain
+Bourqui, Jenny Benois-Pineau, Akka Zemmar`

cam.py

@@ -5,7 +5,7 @@
 import torch
 from torchvision import models
 from pytorch_grad_cam import (
-    GradCAM, HiResCAM, ScoreCAM, GradCAMPlusPlus,
+    GradCAM, FEM, HiResCAM, ScoreCAM, GradCAMPlusPlus,
     AblationCAM, XGradCAM, EigenCAM, EigenGradCAM,
     LayerCAM, FullGrad, GradCAMElementWise, KPCA_CAM
 )
@@ -34,7 +34,7 @@ def get_args():
         'of cam_weights*activations')
     parser.add_argument('--method', type=str, default='gradcam',
                         choices=[
-                            'gradcam', 'hirescam', 'gradcam++',
+                            'gradcam', 'fem', 'hirescam', 'gradcam++',
                             'scorecam', 'xgradcam', 'ablationcam',
                             'eigencam', 'eigengradcam', 'layercam',
                             'fullgrad', 'gradcamelementwise', 'kpcacam'
@@ -73,6 +73,7 @@ def get_args():
         "eigengradcam": EigenGradCAM,
         "layercam": LayerCAM,
         "fullgrad": FullGrad,
+        "fem": FEM,
         "gradcamelementwise": GradCAMElementWise,
         'kpcacam': KPCA_CAM
     }

pytorch_grad_cam/__init__.py

@@ -1,4 +1,5 @@
 from pytorch_grad_cam.grad_cam import GradCAM
+from pytorch_grad_cam.fem import FEM
 from pytorch_grad_cam.hirescam import HiResCAM
 from pytorch_grad_cam.grad_cam_elementwise import GradCAMElementWise
 from pytorch_grad_cam.ablation_layer import AblationLayer, AblationLayerVit, AblationLayerFasterRCNN

pytorch_grad_cam/fem.py

@@ -0,0 +1,45 @@
+import numpy as np
+from pytorch_grad_cam.base_cam import BaseCAM
+
+"""Feature Explanation Method.
+Fuad, K. A. A., Martin, P. E., Giot, R., Bourqui, R., Benois-Pineau, J., & Zemmari, A. (2020, November). Features understanding in 3D CNNS for actions recognition in video. In 2020 Tenth International Conference on Image Processing Theory, Tools and Applications (IPTA) (pp. 1-6). IEEE.
+https://hal.science/hal-02963298/document
+"""
+
+class FEM(BaseCAM):
+    def __init__(self, model, target_layers, 
+                 reshape_transform=None, k=2):
+        super(FEM, self).__init__(model,
+                                       target_layers,
+                                       reshape_transform,
+                                       uses_gradients=False)
+        self.k = k
+
+    def get_cam_image(self,
+                      input_tensor,
+                      target_layer,
+                      target_category,
+                      activations,
+                      grads,
+                      eigen_smooth):
+        
+        
+        # 2D image
+        if len(activations.shape) == 4:
+            axis = (2, 3)
+        # 3D image
+        elif len(activations.shape) == 5:
+            axis = (2, 3, 4)
+        else:
+            raise ValueError("Invalid activations shape." 
+                             "Shape of activations should be 4 (2D image) or 5 (3D image).")
+        means = np.mean(activations, axis=axis)
+        stds = np.std(activations, axis=axis)
+        # k sigma rule:
+        # Add extra dimensions to match activations shape
+        th = means + self.k * stds
+        weights_shape = list(means.shape) + [1] * len(axis)
+        th = th.reshape(weights_shape)
+        binary_mask = activations > th
+        weights = binary_mask.mean(axis=axis)
+        return (weights.reshape(weights_shape) * activations).sum(axis=1)