variable names more general; remove use_cuda

Author: holyseven

interpretdl/evaluate_interpreter/abc_evaluator.py

@@ -14,27 +14,20 @@ class InterpreterEvaluator(ABC):
     All evaluators aim to evaluate the trustworthiness of the interpretation algorithms. Besides theoretical
     verification of the algorithm, here the evaluators validate the trustworthiness by looking through the obtained
     explanations from the interpretation algorithms. Different evaluators are provided.
-    
-    .. warning:: ``use_cuda`` would be deprecated soon. Use ``device`` directly.
     """
 
-    def __init__(self, paddle_model: callable or None, device: str = 'gpu:0', use_cuda: bool = None, **kwargs):
+    def __init__(self, model: callable or None, device: str = 'gpu:0', **kwargs):
         """
 
         Args:
-            paddle_model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
+            model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
                 is not always required if the model is not involved. 
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
                 etc. Again, this is not always required if the model is not involved.
         """
 
-        if use_cuda in [True, False]:
-            warnings.warn('``use_cuda`` would be deprecated soon. '
-                          'Use ``device`` directly.', stacklevel=2)
-            device = 'gpu' if use_cuda and device[:3] == 'gpu' else 'cpu'
-
         self.device = device
-        self.paddle_model = paddle_model
+        self.model = model
         self.predict_fn = None
 
     def _build_predict_fn(self, rebuild: bool = False):
@@ -52,7 +45,7 @@ def _build_predict_fn(self, rebuild: bool = False):
             paddle.set_device(self.device)
 
             # to get gradients, the ``train`` mode must be set.
-            self.paddle_model.eval()
+            self.model.eval()
 
             def predict_fn(inputs):
                 """predict_fn for input gradients based interpreters,
@@ -69,7 +62,7 @@ def predict_fn(inputs):
                 with paddle.no_grad():
                     inputs = tuple(paddle.to_tensor(inp) for inp in inputs) if isinstance(inputs, tuple) \
                         else (paddle.to_tensor(inputs), )
-                    logits = self.paddle_model(*inputs)  # get logits, [bs, num_c]
+                    logits = self.model(*inputs)  # get logits, [bs, num_c]
                     probas = paddle.nn.functional.softmax(logits, axis=1)  # get probabilities.
                 return probas.numpy()
 

interpretdl/evaluate_interpreter/deletion_insertion.py

@@ -26,26 +26,25 @@ class DeletionInsertion(InterpreterEvaluator):
     """
 
     def __init__(self,
-                 paddle_model: callable,
+                 model: callable,
                  device: str,
-                 use_cuda: bool = None,
                  compute_deletion: bool = True,
                  compute_insertion: bool = True,
                  **kwargs):
         """
 
         Args:
-            paddle_model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
+            model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
                 is not always required if the model is not involved. 
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
                 etc. Again, this is not always required if the model is not involved.
             compute_deletion (bool, optional): Whether compute deletion score. Defaults to ``True``.
             compute_insertion (bool, optional): Whether compute insertion score. Defaults to ``True``.
 
         Raises:
             ValueError: At least one of ``compute_deletion`` and ``compute_insertion`` must be True.
         """
-        super().__init__(paddle_model, device, use_cuda, **kwargs)
+        super().__init__(model, device, **kwargs)
 
         if (not compute_deletion) or (not compute_insertion):
             raise ValueError('At least one of ``compute_deletion`` and ``compute_insertion`` must be True.')

interpretdl/evaluate_interpreter/infidelity.py

@@ -33,22 +33,22 @@ class Infidelity(InterpreterEvaluator):
     pixels; while the difference (the latter term) should also be large because the model depends on important pixels
     to make decisions. Like this, large values would be offset by large values if the explanation is faithful to the 
     model. Otherwise, for uniform explanations (all being constant), the former term would be a constant value and the
-    infidelity would become large。
+    infidelity would become large.
 
     More details about the measure can be found in the original paper: https://arxiv.org/abs/1901.09392.
     """
     def __init__(self,
-                paddle_model: callable,
+                model: callable,
                 device: str = 'gpu:0',
                 **kwargs):
         """
 
         Args:
-            paddle_model (callable): _description_
+            model (callable): _description_
             device (_type_, optional): _description_. Defaults to 'gpu:0'.
         """
 
-        super().__init__(paddle_model, device, None, **kwargs)
+        super().__init__(model, device, **kwargs)
         self.results = {}
 
     def _build_predict_fn(self, rebuild: bool = False):
@@ -74,7 +74,7 @@ def _build_predict_fn(self, rebuild: bool = False):
             paddle.set_device(self.device)
 
             # to get gradients, the ``train`` mode must be set.
-            self.paddle_model.eval()
+            self.model.eval()
 
             def predict_fn(data):
                 """predict_fn for input gradients based interpreters,
@@ -91,7 +91,7 @@ def predict_fn(data):
                 with paddle.no_grad():
                     # Follow the `official implementation <https://github.com/chihkuanyeh/saliency_evaluation>`_
                     # to use logits as output.
-                    logits = self.paddle_model(paddle.to_tensor(data))  # get logits, [bs, num_c]
+                    logits = self.model(paddle.to_tensor(data))  # get logits, [bs, num_c]
                     # probas = paddle.nn.functional.softmax(logits, axis=1)  # get probabilities.
                 return logits.numpy()
 
@@ -234,34 +234,92 @@ def evaluate(self,
 
 
 class InfidelityNLP(InterpreterEvaluator):
-    def __init__(self, paddle_model: callable or None, device: str = 'gpu:0', **kwargs):
-        super().__init__(paddle_model, device, **kwargs)
+    def __init__(self, model: callable or None, device: str = 'gpu:0', **kwargs):
+        super().__init__(model, device, **kwargs)
         self.results = {}
 
-    def _generate_samples(self, input_ids, masked_id=0):
+    def _generate_samples(self, input_ids, masked_id: int, is_random_samples: bool):
         num_tokens = len(input_ids)
 
-        # like 1d-conv, stride=1, kernel-size={1,2,3,4,5}
-        generated_samples = []
-        input_ids_array = np.array([input_ids])
-        for ks in range(1, 6):
-            if ks > num_tokens - 2:
-                break
-            for i in range(1, num_tokens-ks):
-                tmp = np.copy(input_ids_array)
-                tmp[0, i:i+ks] = masked_id
-                generated_samples.append(tmp)
-        
-        perturbed_samples = np.concatenate(generated_samples, axis=0)
-        Is = perturbed_samples != input_ids_array
-
-        return perturbed_samples, Is
-
-    def evaluate(self, raw_text: str, explanation: list or np.ndarray, tokenizer: callable, recompute: bool = False):
+        if is_random_samples:
+            # This is more suitable for long documents.
+            # we concat three kinds of perturbations: 
+            # randomly perturbing 1%, 2%, 3%, 4% or 5% tokens respectively
+            # with 40 times
+            num_repeats = 40
+            results = []
+            ids_array = np.array([input_ids]*num_repeats)
+            for p in range(1, 6):
+                _k = int(num_tokens * p / 100)
+
+                # not choose from {0, -1}, i.e., [CLS] and [SEP]
+                # https://stackoverflow.com/a/53893160/4834515
+                pert_k = np.random.rand(num_repeats, num_tokens-2).argpartition(_k, axis=1)[:,:_k] + 1
+
+                pert_array = np.copy(ids_array)
+                # vectorized slicing.
+                # https://stackoverflow.com/a/74024396/4834515
+                row_indexes = np.arange(num_repeats)[:, None]
+                pert_array[row_indexes, pert_k] = masked_id
+
+                results.append(pert_array)
+
+            perturbed_samples = np.concatenate(results)  # [200, num_tokens]
+            Is = perturbed_samples != np.array([input_ids])  # [200, num_tokens]
+
+            return perturbed_samples, Is
+        else:
+            # This is more suitable for short documents.
+            # like 1d-conv, stride=1, kernel-size={1,2,3,4,5}
+            generated_samples = []
+            input_ids_array = np.array([input_ids])
+            for ks in range(1, 6):
+                if ks > num_tokens - 2:
+                    break
+                for i in range(1, num_tokens-ks):
+                    tmp = np.copy(input_ids_array)
+                    tmp[0, i:i+ks] = masked_id
+                    generated_samples.append(tmp)
+            
+            perturbed_samples = np.concatenate(generated_samples, axis=0)
+            Is = perturbed_samples != input_ids_array
+
+            return perturbed_samples, Is
+
+    # def _generate_samples(self, input_ids, masked_id=0):
+    #     num_tokens = len(input_ids)
+
+    #     # we concat three kinds of perturbations: 
+    #     # randomly perturbing 1, 2 or 3 tokens respectively
+    #     # with 33 times
+    #     num_repeats = 33
+
+    #     ids_array = np.array([input_ids]*num_repeats)
+
+    #     # not choose from {0, -1}, [CLS] and [SEP]
+    #     # https://stackoverflow.com/a/53893160/4834515
+    #     pert_1 = np.random.rand(num_repeats, num_tokens-2).argpartition(1, axis=1)[:,:1] + 1
+    #     pert_2 = np.random.rand(num_repeats, num_tokens-2).argpartition(2, axis=1)[:,:2] + 1
+    #     pert_3 = np.random.rand(num_repeats, num_tokens-2).argpartition(3, axis=1)[:,:3] + 1
+
+    #     pert_1_array = np.copy(ids_array)
+    #     pert_2_array = np.copy(ids_array)
+    #     pert_3_array = np.copy(ids_array)
+
+    #     # https://stackoverflow.com/a/74024396/4834515
+    #     row_indexes = np.arange(num_repeats)[:, None]
+    #     pert_1_array[row_indexes, pert_1] = masked_id
+    #     pert_2_array[row_indexes, pert_2] = masked_id
+    #     pert_3_array[row_indexes, pert_3] = masked_id
+
+    #     perturbed_samples = np.concatenate([pert_1_array, pert_2_array, pert_3_array])
+    #     return perturbed_samples, perturbed_samples != ids_array
+
+    def evaluate(self, raw_text: str, explanation: list or np.ndarray, tokenizer: callable, max_seq_len=128, is_random_samples=False, recompute: bool = False):
         self._build_predict_fn()
 
         # tokenizer text to ids
-        encoded_inputs = tokenizer(raw_text, max_seq_len=128)
+        encoded_inputs = tokenizer(raw_text, max_seq_len=max_seq_len)
         # order is important. *_batched_and_to_tuple will be the input for the model.
         _batched_and_to_tuple = tuple([np.array([v]) for v in encoded_inputs.values()])
 
@@ -276,7 +334,7 @@ def evaluate(self, raw_text: str, explanation: list or np.ndarray, tokenizer: ca
         # generate perturbation samples.
         if 'proba_diff' not in self.results or recompute:
             ## x and I related.
-            generated_samples, Is = self._generate_samples(encoded_inputs['input_ids'], tokenizer.pad_token_id)
+            generated_samples, Is = self._generate_samples(encoded_inputs['input_ids'], tokenizer.pad_token_id, is_random_samples)
             self.results['generated_samples'] = generated_samples
             self.results['Is'] = Is
             proba_pert = self.predict_fn(generated_samples)[:, label]

interpretdl/evaluate_interpreter/perturbation.py

@@ -30,25 +30,25 @@ class Perturbation(InterpreterEvaluator):
     """
 
     def __init__(self,
-                 paddle_model: callable,
+                 model: callable,
                  device: str = 'gpu:0',
                  compute_MoRF: bool = True,
                  compute_LeRF: bool = True,
                  **kwargs):
         """_summary_
 
         Args:
-            paddle_model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
+            model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
                 is not always required if the model is not involved. 
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
                 etc. Again, this is not always required if the model is not involved.
             compute_MoRF (bool, optional): Whether comptue MoRF score. Defaults to True.
             compute_LeRF (bool, optional): Whether comptue LeRF score. Defaults to True.
 
         Raises:
             ValueError: 'At least one of ``compute_MoRF`` and ``compute_LeRF`` must be True.'
         """
-        super().__init__(paddle_model, device, None, **kwargs)
+        super().__init__(model, device, **kwargs)
 
         if (not compute_MoRF) and (not compute_LeRF):
             raise ValueError('At least one of ``compute_MoRF`` and ``compute_LeRF`` must be True.')
@@ -281,23 +281,23 @@ class PerturbationNLP(InterpreterEvaluator):
     """
 
     def __init__(self,
-                 paddle_model: callable,
+                 model: callable,
                  device: str = 'gpu:0',
                  **kwargs):
         """_summary_
 
         Args:
-            paddle_model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
+            model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions. This 
                 is not always required if the model is not involved. 
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
                 etc. Again, this is not always required if the model is not involved.
             compute_MoRF (bool, optional): Whether comptue MoRF score. Defaults to True.
             compute_LeRF (bool, optional): Whether comptue LeRF score. Defaults to True.
 
         Raises:
             ValueError: 'At least one of ``compute_MoRF`` and ``compute_LeRF`` must be True.'
         """
-        super().__init__(paddle_model, device, None, **kwargs)
+        super().__init__(model, device, None, **kwargs)
         self._build_predict_fn()
 
     def evaluate(self,

interpretdl/interpreter/_normlime_base.py

@@ -20,15 +20,15 @@ class NormLIMECVInterpreter(LIMECVInterpreter):
 
     """
 
-    def __init__(self, paddle_model: callable, device: str = 'gpu:0', use_cuda=None):
+    def __init__(self, model: callable, device: str = 'gpu:0'):
         """
 
         Args:
-            paddle_model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions.
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` etc.
+            model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions.
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` etc.
         """
 
-        LIMECVInterpreter.__init__(self, paddle_model, use_cuda=use_cuda, device=device)
+        LIMECVInterpreter.__init__(self, model, device)
         self.lime_interpret = super().interpret
 
     def _get_lime_weights(self, data, num_samples, batch_size, save=False):
@@ -188,14 +188,14 @@ class NormLIMENLPInterpreter(LIMENLPInterpreter):
 
     """
 
-    def __init__(self, paddle_model: callable, device: str = 'gpu:0', use_cuda=None):
+    def __init__(self, model: callable, device: str = 'gpu:0'):
         """
 
         Args:
-            paddle_model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions.
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` etc.
+            model (callable): A model with :py:func:`forward` and possibly :py:func:`backward` functions.
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` etc.
         """
-        LIMENLPInterpreter.__init__(self, paddle_model, device, use_cuda)
+        LIMENLPInterpreter.__init__(self, model, device)
         self.lime_interpret = super().interpret
 
     def _get_lime_weights(self, data, preprocess_fn, num_samples, batch_size, unk_id, pad_id, lod_levels, save=False):

interpretdl/interpreter/abc_interpreter.py

@@ -541,7 +541,7 @@ def __init__(self, model: callable, device: str = 'gpu:0') -> None:
             device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
                 etc.
         """
-        Interpreter.__init__(self, model, device, None)
+        Interpreter.__init__(self, model, device)
 
     def _build_predict_fn(self, rebuild=False, layer_name='word_embeddings', gradient_of='probability'):
 

interpretdl/interpreter/consensus.py

@@ -1,5 +1,9 @@
 import numpy as np
-from .abc_interpreter import Interpreter
+
+try:
+    from .abc_interpreter_m import Interpreter
+except:
+    from .abc_interpreter import Interpreter
 
 
 class ConsensusInterpreter(object):
@@ -16,21 +20,20 @@ class ConsensusInterpreter(object):
     `PPClas <https://github.com/PaddlePaddle/PaddleClas/blob/release/2.3/ppcls/arch/backbone/__init__.py>`_.
     """
 
-    def __init__(self, InterpreterClass, list_of_models: list, device: str = 'gpu:0', use_cuda=None, **kwargs):
+    def __init__(self, InterpreterClass, list_of_models: list, device: str = 'gpu:0', **kwargs):
         """
         
         Args:
             InterpreterClass ([type]): The given Interpreter defined in InterpretDL.
             list_of_models (list): a list of trained models.
-            device (str): The device used for running ``paddle_model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
+            device (str): The device used for running ``model``, options: ``"cpu"``, ``"gpu:0"``, ``"gpu:1"`` 
                 etc.
         """
         assert issubclass(InterpreterClass, Interpreter)
 
         self.InterpreterClass = InterpreterClass
         self.list_of_models = list_of_models
         self.device = device
-        self.use_cuda = use_cuda
         self.other_args = kwargs
 
     def interpret(self, inputs: str or list(str) or np.ndarray, **kwargs) -> np.ndarray:
@@ -86,7 +89,7 @@ def interpret(self, inputs: str or list(str) or np.ndarray, **kwargs) -> np.ndar
 
         exps = []
         for model in self.list_of_models:
-            interpreter = self.InterpreterClass(model, self.device, self.use_cuda, **self.other_args)
+            interpreter = self.InterpreterClass(model, self.device, **self.other_args)
             raw_explanation = interpreter.interpret(inputs, visual=False, save_path=None, **kwargs)
             exps.append(raw_explanation)