close(#42) Feature sequence (#43)

* gradcam available in sequence(text)

Author: sssshwan

.gitignore

@@ -103,5 +103,7 @@ ENV/
 *.swp
 cifar-10-batches-py/
 *.tar.gz
+*.tar
 tmp/
 *.ckpt
+test/data/sequence/

darkon/gradcam/candidate_ops.py

@@ -15,20 +15,33 @@
 import tensorflow as tf
 import numpy as np
 
-_unusable_op_names = ('Shape', 'Reshape', 'Slice', 'Pack', 'Cast', 'ConcatV2')
+_unusable_op_names = (
+    'Shape',
+    'Reshape',
+    'Slice',
+    'Pack',
+    'Cast',
+    'ConcatV2',
+    'Const',
+    'Identity',
+    'ZerosLike',
+    'Assign',
+    'VariableV2')
 
 
 def _unusable_ops(op):
     if len(op.outputs) == 0 \
             or 'save' in op.name \
+            or 'gradients/' in op.name \
+            or '/Initializer' in op.name \
             or op.op_def is None \
             or op.op_def.name in _unusable_op_names:
         return True
     else:
         return False
 
 
-def candidate_featuremap_op_names(sess, graph):
+def candidate_featuremap_op_names(sess, graph, feed_options):
     operations = []
     out_ranks = []
     out_shapes = []
@@ -41,32 +54,36 @@ def candidate_featuremap_op_names(sess, graph):
         out_shapes.append(tf.shape(op.outputs[0]))
         operations.append(op)
 
-    out_ranks_val, out_shapes_val = sess.run([out_ranks, out_shapes])
+    out_ranks_val, out_shapes_val = sess.run([out_ranks, out_shapes], feed_dict=feed_options)
 
     ret = []
     for out_rank, out_shape, op in zip(out_ranks_val, out_shapes_val, operations):
-        if out_rank != 4 or out_shape[1] == 1 or out_shape[2] == 1 or out_shape[0] != 1:
+        if out_rank != 4 or (out_shape[1] == 1 and out_shape[2] == 1) or out_shape[0] != 1:
             continue
 
         ret.append(op.name)
     return ret
 
 
-def candidate_predict_op_names(sess, num_classes, graph):
+def candidate_predict_op_names(sess, num_classes, graph, feed_options):
     operations = []
+    out_ranks = []
     out_shapes = []
 
     for op in graph.get_operations():
         if _unusable_ops(op):
             continue
 
+        out_ranks.append(tf.rank(op.outputs[0]))
         out_shapes.append(tf.shape(op.outputs[0]))
         operations.append(op)
 
-    out_shapes_val = sess.run(out_shapes)
+    out_ranks_val, out_shapes_val = sess.run([out_ranks, out_shapes], feed_dict=feed_options)
 
     ret = []
-    for out_shape, op in zip(out_shapes_val, operations):
+    for out_rank, out_shape, op in zip(out_ranks_val, out_shapes_val, operations):
+        if out_rank == 1:
+            continue
         if np.prod(out_shape) != num_classes:
             continue
 

darkon/gradcam/gradcam.py

@@ -27,6 +27,7 @@
 import numpy as np
 import tensorflow as tf
 import cv2
+from skimage.transform import resize as skimage_resize
 
 from .guided_grad import replace_grad_to_guided_grad
 from .candidate_ops import candidate_featuremap_op_names, candidate_predict_op_names
@@ -76,10 +77,11 @@ class Gradcam:
     def __init__(self, x_placeholder, num_classes, featuremap_op_name, predict_op_name=None, graph=None):
         self._x_placeholder = x_placeholder
         graph = graph if graph is not None else tf.get_default_graph()
+        self.graph = graph
 
         predict_op_name = self._find_prob_layer(predict_op_name, graph)
-        self._prob_ts = graph.get_operation_by_name(predict_op_name).outputs
-        self._target_ts = graph.get_operation_by_name(featuremap_op_name).outputs
+        self._prob_ts = graph.get_operation_by_name(predict_op_name).outputs[0]
+        self._target_ts = graph.get_operation_by_name(featuremap_op_name).outputs[0]
 
         self._class_idx = tf.placeholder(tf.int32)
         top1 = tf.argmax(tf.reshape(self._prob_ts, [-1]))
@@ -91,10 +93,10 @@ def __init__(self, x_placeholder, num_classes, featuremap_op_name, predict_op_na
 
         replace_grad_to_guided_grad(graph)
 
-        max_output = tf.reduce_max(self._target_ts, axis=3)
+        max_output = tf.reduce_max(self._target_ts, axis=2)
         self._saliency_map = tf.gradients(tf.reduce_sum(max_output), x_placeholder)[0]
 
-    def gradcam(self, sess, input_data, target_index=None):
+    def gradcam(self, sess, input_data, target_index=None, feed_options=dict()):
         """ Calculate Grad-CAM (class activation map) and Guided Grad-CAM for given input on target class
 
         Parameters
@@ -106,6 +108,8 @@ def gradcam(self, sess, input_data, target_index=None):
         target_index : int
             Target class index
             If None, predicted class index is used
+        feed_options : dict
+            Optional parameters to graph
 
         Returns
         -------
@@ -120,41 +124,54 @@ def gradcam(self, sess, input_data, target_index=None):
             * guided_backprop: Guided backprop result
 
         """
-
         input_feed = np.expand_dims(input_data, axis=0)
-        image_height, image_width = input_data.shape[:2]
+        if input_data.ndim == 3:
+            is_image = True
+            image_height, image_width = input_data.shape[:2]
+        if input_data.ndim == 1:
+            is_image = False
+            input_length = input_data.shape[0]
 
         if target_index is not None:
-            conv_out_eval, grad_eval = sess.run(
-                [self._target_ts, self._grad_by_idx],
-                feed_dict={self._x_placeholder: input_feed, self._class_idx: target_index})
+            feed_dict = {self._x_placeholder: input_feed, self._class_idx: target_index}
+            feed_dict.update(feed_options)
+            conv_out_eval, grad_eval = sess.run([self._target_ts, self._grad_by_idx], feed_dict=feed_dict)
         else:
-            conv_out_eval, grad_eval = sess.run(
-                [self._target_ts, self._grad_by_top1],
-                feed_dict={self._x_placeholder: input_feed})
+            feed_dict = {self._x_placeholder: input_feed}
+            feed_dict.update(feed_options)
+            conv_out_eval, grad_eval = sess.run([self._target_ts, self._grad_by_top1], feed_dict=feed_dict)
 
         weights = np.mean(grad_eval, axis=(0, 1, 2))
         conv_out_eval = np.squeeze(conv_out_eval, axis=0)
-        cam = np.ones(conv_out_eval.shape[:2], dtype=np.float32)
+        cam = np.zeros(conv_out_eval.shape[:2], dtype=np.float32)
 
         for i, w in enumerate(weights):
             cam += w * conv_out_eval[:, :, i]
-        cam = cv2.resize(cam, (image_height, image_width))
+
+        if is_image:
+            cam += 1
+            cam = cv2.resize(cam, (image_height, image_width))
+            saliency_val = sess.run(self._saliency_map, feed_dict={self._x_placeholder: input_feed})
+            saliency_val = np.squeeze(saliency_val, axis=0)
+        else:
+            cam = skimage_resize(cam, (input_length, 1), preserve_range=True, mode='reflect')
+            cam = np.transpose(cam)
+
         cam = np.maximum(cam, 0)
         heatmap = cam / np.max(cam)
 
-        saliency_val = sess.run(self._saliency_map, feed_dict={self._x_placeholder: input_feed})
-        saliency_val = np.squeeze(saliency_val, axis=0)
+        ret = {'heatmap': heatmap}
 
-        return {
-            'gradcam_img': self.overlay_gradcam(input_data, heatmap),
-            'guided_gradcam_img': _deprocess_image(saliency_val * heatmap[..., None]),
-            'heatmap': heatmap,
-            'guided_backprop': saliency_val
-        }
+        if is_image:
+            ret.update({
+                'gradcam_img': self.overlay_gradcam(input_data, heatmap),
+                'guided_gradcam_img': _deprocess_image(saliency_val * heatmap[..., None]),
+                'guided_backprop': saliency_val
+            })
+        return ret
 
     @staticmethod
-    def candidate_featuremap_op_names(sess, graph=None):
+    def candidate_featuremap_op_names(sess, graph=None, feed_options=dict()):
         """ Returns the list of candidates for operation names of CNN feature map layer
 
         Parameters
@@ -163,17 +180,19 @@ def candidate_featuremap_op_names(sess, graph=None):
             Tensorflow session
         graph: tf.Graph
             Tensorflow graph
+        feed_options: dict
+            Optional parameters to graph
         Returns
         -------
         list
             String list of candidates
 
         """
         graph = graph if graph is not None else tf.get_default_graph()
-        return candidate_featuremap_op_names(sess, graph)
+        return candidate_featuremap_op_names(sess, graph, feed_options)
 
     @staticmethod
-    def candidate_predict_op_names(sess, num_classes, graph=None):
+    def candidate_predict_op_names(sess, num_classes, graph=None, feed_options=dict()):
         """ Returns the list of candidate for operation names of prediction layer
 
         Parameters
@@ -184,14 +203,16 @@ def candidate_predict_op_names(sess, num_classes, graph=None):
             Number of prediction classes
         graph: tf.Graph
             Tensorflow graph
+        feed_options: dict
+            Optional parameters to graph
         Returns
         -------
         list
             String list of candidates
 
         """
         graph = graph if graph is not None else tf.get_default_graph()
-        return candidate_predict_op_names(sess, num_classes, graph)
+        return candidate_predict_op_names(sess, num_classes, graph, feed_options)
 
     @staticmethod
     def overlay_gradcam(image, heatmap):

download-models.sh

@@ -2,6 +2,10 @@ test_model_dir="./test/data"
 
 wget http://download.tensorflow.org/models/resnet_v1_50_2016_08_28.tar.gz
 wget http://download.tensorflow.org/models/vgg_16_2016_08_28.tar.gz
+wget https://raw.githubusercontent.com/darkonhub/darkon-examples/master/gradcam/sequence.tar
+
+mkdir -p $test_model_dir
 tar -xf resnet_v1_50_2016_08_28.tar.gz -C $test_model_dir
 tar -xf vgg_16_2016_08_28.tar.gz -C $test_model_dir
+tar -xf sequence.tar -C $test_model_dir
 

requirements.txt

@@ -1,3 +1,4 @@
 numpy
 six
-opencv-python
+opencv-python
+scikit-image

test/test_gradcam.py

@@ -101,6 +101,8 @@ def test_resnet(self):
         self.graph_origin = tf.get_default_graph()
         self.target_op_name = darkon.Gradcam.candidate_featuremap_op_names(sess, self.graph_origin)[-1]
         self.model_name = 'resnet'
+        
+        self.assertEqual('resnet_v1_50/block4/unit_3/bottleneck_v1/Relu', self.target_op_name) 
 
     def test_vgg(self):
         with slim.arg_scope(vgg.vgg_arg_scope()):
@@ -116,3 +118,4 @@ def test_vgg(self):
         self.graph_origin = tf.get_default_graph()
         self.target_op_name = darkon.Gradcam.candidate_featuremap_op_names(sess, self.graph_origin)[-2]
         self.model_name = 'vgg'
+        self.assertEqual('vgg_16/conv5/conv5_3/Relu', self.target_op_name)

test/test_gradcam_sequence.py

@@ -0,0 +1,53 @@
+# Copyright 2017 Neosapience, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ========================================================================
+import unittest
+
+import darkon
+import tensorflow as tf
+import numpy as np
+from tensorflow.contrib import learn
+
+class TestGradcamSequence(unittest.TestCase):
+    def setUp(self):
+        tf.reset_default_graph()
+        x_raw = ["a masterpiece of four years in the making"]
+        vocab_path = "test/data/sequence/vocab"
+        vocab_processor = learn.preprocessing.VocabularyProcessor.restore(vocab_path)
+        self.x_test_batch = np.array(list(vocab_processor.transform(x_raw)))
+        self.y_test_batch = [[1.0, 0.0]]
+        
+    def test_text(self):
+        sess = tf.InteractiveSession()
+        checkpoint_file = "test/data/sequence/model-15000"
+        saver = tf.train.import_meta_graph("{}.meta".format(checkpoint_file))
+        saver.restore(sess, checkpoint_file)
+        graph = tf.get_default_graph()   
+        input_x = graph.get_operation_by_name("input_x").outputs[0]
+        dropout_keep_prob = graph.get_operation_by_name("dropout_keep_prob").outputs[0]
+        input_y = graph.get_operation_by_name("input_y").outputs[0]
+
+        conv_op_names = darkon.Gradcam.candidate_featuremap_op_names(sess, 
+            feed_options={input_x: self.x_test_batch, input_y: self.y_test_batch ,dropout_keep_prob:1.0})
+                
+        prob_op_names = darkon.Gradcam.candidate_predict_op_names(sess, 2, 
+            feed_options={input_x: self.x_test_batch, input_y: self.y_test_batch ,dropout_keep_prob:1.0})
+        
+        conv_name = conv_op_names[-7]
+        prob_name = prob_op_names[-1]
+        self.assertEqual(conv_name, "conv-maxpool-3/relu")
+        self.assertEqual(prob_name, "output/scores")
+            
+        insp = darkon.Gradcam(input_x, 2, conv_name, prob_name, graph=graph)
+        ret = insp.gradcam(sess, self.x_test_batch[0], feed_options={dropout_keep_prob: 1})