Merge pull request #1036 from YiwenShaoStephen/pytorch_espresso

Add PyTorchEspresso estimator to support Espresso models for ASR

Author: beat-buesser

AUTHORS

@@ -14,4 +14,5 @@
 - AGH University of Science and Technology
 - Rensselaer Polytechnic Institute (RPI)
 - IMT Atlantique
+- Johns Hopkins University
 - Troj.AI

art/attacks/evasion/fast_gradient.py

@@ -409,17 +409,22 @@ def _compute_perturbation(
                     )
 
         # Check for NaN before normalisation an replace with 0
-        if np.isnan(grad).any():
+        if grad.dtype != np.object and np.isnan(grad).any():
             logger.warning("Elements of the loss gradient are NaN and have been replaced with 0.0.")
             grad = np.where(np.isnan(grad), 0.0, grad)
+        else:
+            for i, _ in enumerate(grad):
+                grad_i_array = grad[i].astype(np.float32)
+                if np.isnan(grad_i_array).any():
+                    grad[i] = np.where(np.isnan(grad_i_array), 0.0, grad_i_array).astype(np.object)
 
         # Apply mask
         if mask is not None:
             grad = np.where(mask == 0.0, 0.0, grad)
 
         # Apply norm bound
         def _apply_norm(grad, object_type=False):
-            if np.isinf(grad).any():
+            if (grad.dtype != np.object and np.isinf(grad).any()) or np.isnan(grad.astype(np.float32)).any():
                 logger.info("The loss gradient array contains at least one positive or negative infinity.")
 
             if self.norm in [np.inf, "inf"]:
@@ -454,7 +459,16 @@ def _apply_perturbation(
     ) -> np.ndarray:
 
         perturbation_step = eps_step * perturbation
-        perturbation_step[np.isnan(perturbation_step)] = 0
+        if perturbation_step.dtype != np.object:
+            perturbation_step[np.isnan(perturbation_step)] = 0
+        else:
+            for i, _ in enumerate(perturbation_step):
+                perturbation_step_i_array = perturbation_step[i].astype(np.float32)
+                if np.isnan(perturbation_step_i_array).any():
+                    perturbation_step[i] = np.where(
+                        np.isnan(perturbation_step_i_array), 0.0, perturbation_step_i_array
+                    ).astype(np.object)
+
         batch = batch + perturbation_step
         if self.estimator.clip_values is not None:
             clip_min, clip_max = self.estimator.clip_values

art/attacks/evasion/imperceptible_asr/imperceptible_asr_pytorch.py

@@ -31,7 +31,6 @@
 import scipy
 
 from art.attacks.attack import EvasionAttack
-from art.estimators.estimator import BaseEstimator, LossGradientsMixin, NeuralNetworkMixin
 from art.estimators.pytorch import PyTorchEstimator
 from art.estimators.speech_recognition.pytorch_deep_speech import PyTorchDeepSpeech
 from art.estimators.speech_recognition.speech_recognizer import SpeechRecognizerMixin
@@ -78,13 +77,9 @@ class ImperceptibleASRPyTorch(EvasionAttack):
     ]
 
     _estimator_requirements = (
-        BaseEstimator,
-        LossGradientsMixin,
-        NeuralNetworkMixin,
-        SpeechRecognizerMixin,
         PytorchSpeechRecognizerMixin,
+        SpeechRecognizerMixin,
         PyTorchEstimator,
-        PyTorchDeepSpeech,
     )
 
     def __init__(

art/estimators/speech_recognition/__init__.py

@@ -4,4 +4,5 @@
 from art.estimators.speech_recognition.speech_recognizer import SpeechRecognizerMixin
 
 from art.estimators.speech_recognition.pytorch_deep_speech import PyTorchDeepSpeech
+from art.estimators.speech_recognition.pytorch_espresso import PyTorchEspresso
 from art.estimators.speech_recognition.tensorflow_lingvo import TensorFlowLingvoASR

art/estimators/speech_recognition/pytorch_deep_speech.py

@@ -28,8 +28,7 @@
 
 from art import config
 from art.estimators.pytorch import PyTorchEstimator
-from art.estimators.speech_recognition.speech_recognizer import SpeechRecognizerMixin
-from art.estimators.speech_recognition.speech_recognizer import PytorchSpeechRecognizerMixin
+from art.estimators.speech_recognition.speech_recognizer import SpeechRecognizerMixin, PytorchSpeechRecognizerMixin
 from art.utils import get_file
 
 if TYPE_CHECKING: