Merge pull request #603 from Trusted-AI/development_issue_488

Update progress bars in PGDs and FBF adversarial trainer

Author: beat-buesser

art/attacks/evasion/carlini.py

@@ -108,7 +108,7 @@ def __init__(
         :param max_halving: Maximum number of halving steps in the line search optimization.
         :param max_doubling: Maximum number of doubling steps in the line search optimization.
         :param batch_size: Size of the batch on which adversarial samples are generated.
-        :param verbose: Indicates whether to print verbose messages.
+        :param verbose: Show progress bars.
         """
         super().__init__(estimator=classifier)
 
@@ -523,7 +523,7 @@ def __init__(
         :param max_doubling: Maximum number of doubling steps in the line search optimization.
         :param eps: An upper bound for the L_0 norm of the adversarial perturbation.
         :param batch_size: Size of the batch on which adversarial samples are generated.
-        :param verbose: Indicates whether to print verbose messages.
+        :param verbose: Show progress bars.
         """
         super().__init__(estimator=classifier)
 

art/attacks/evasion/deepfool.py

@@ -74,6 +74,7 @@ def __init__(
         :param nb_grads: The number of class gradients (top nb_grads w.r.t. prediction) to compute. This way only the
                          most likely classes are considered, speeding up the computation.
         :param batch_size: Batch size
+        :param verbose: Show progress bars.
         """
         super().__init__(estimator=classifier)
         self.max_iter = max_iter

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent.py

@@ -69,6 +69,7 @@ class ProjectedGradientDescent(EvasionAttack):
         "minimal",
         "max_iter",
         "random_eps",
+        "verbose",
     ]
 
     _estimator_requirements = (BaseEstimator, LossGradientsMixin)
@@ -84,6 +85,7 @@ def __init__(
         num_random_init: int = 0,
         batch_size: int = 32,
         random_eps: bool = False,
+        verbose: bool = True,
     ):
         """
         Create a :class:`.ProjectedGradientDescent` instance.
@@ -101,6 +103,7 @@ def __init__(
         :param num_random_init: Number of random initialisations within the epsilon ball. For num_random_init=0 starting
                                 at the original input.
         :param batch_size: Size of the batch on which adversarial samples are generated.
+        :param verbose: Show progress bars.
         """
         super().__init__(estimator=estimator)
 
@@ -112,6 +115,7 @@ def __init__(
         self.num_random_init = num_random_init
         self.batch_size = batch_size
         self.random_eps = random_eps
+        self.verbose = verbose
         ProjectedGradientDescent._check_params(self)
 
         no_preprocessing = self.estimator.preprocessing is None or (
@@ -133,6 +137,7 @@ def __init__(
                 num_random_init=num_random_init,
                 batch_size=batch_size,
                 random_eps=random_eps,
+                verbose=verbose,
             )
 
         elif isinstance(self.estimator, TensorFlowV2Classifier) and no_preprocessing and no_defences:
@@ -146,6 +151,7 @@ def __init__(
                 num_random_init=num_random_init,
                 batch_size=batch_size,
                 random_eps=random_eps,
+                verbose=verbose,
             )
 
         else:
@@ -159,6 +165,7 @@ def __init__(
                 num_random_init=num_random_init,
                 batch_size=batch_size,
                 random_eps=random_eps,
+                verbose=verbose,
             )
 
     def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
@@ -207,3 +214,6 @@ def _check_params(self) -> None:
 
         if self.max_iter <= 0:
             raise ValueError("The number of iterations `max_iter` has to be a positive integer.")
+
+        if not isinstance(self.verbose, bool):
+            raise ValueError("The verbose has to be a Boolean.")

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_numpy.py

@@ -30,6 +30,7 @@
 
 import numpy as np
 from scipy.stats import truncnorm
+from tqdm import trange
 
 from art.attacks.evasion.fast_gradient import FastGradientMethod
 from art.config import ART_NUMPY_DTYPE
@@ -57,7 +58,7 @@ class ProjectedGradientDescentCommon(FastGradientMethod):
     | Paper link: https://arxiv.org/abs/1706.06083
     """
 
-    attack_params = FastGradientMethod.attack_params + ["max_iter", "random_eps"]
+    attack_params = FastGradientMethod.attack_params + ["max_iter", "random_eps", "verbose"]
     _estimator_requirements = (BaseEstimator, LossGradientsMixin)
 
     def __init__(
@@ -71,6 +72,7 @@ def __init__(
         num_random_init: int = 0,
         batch_size: int = 32,
         random_eps: bool = False,
+        verbose: bool = True,
     ) -> None:
         """
         Create a :class:`.ProjectedGradientDescentCommon` instance.
@@ -88,6 +90,7 @@ def __init__(
         :param num_random_init: Number of random initialisations within the epsilon ball. For num_random_init=0
             starting at the original input.
         :param batch_size: Size of the batch on which adversarial samples are generated.
+        :param verbose: Show progress bars.
         """
         super().__init__(
             estimator=estimator,  # type: ignore
@@ -101,6 +104,7 @@ def __init__(
         )
         self.max_iter = max_iter
         self.random_eps = random_eps
+        self.verbose = verbose
         ProjectedGradientDescentCommon._check_params(self)
 
         if self.random_eps:
@@ -204,6 +208,7 @@ def __init__(
         num_random_init: int = 0,
         batch_size: int = 32,
         random_eps: bool = False,
+        verbose: bool = True,
     ) -> None:
         """
         Create a :class:`.ProjectedGradientDescentNumpy` instance.
@@ -221,6 +226,7 @@ def __init__(
         :param num_random_init: Number of random initialisations within the epsilon ball. For num_random_init=0 starting
                                 at the original input.
         :param batch_size: Size of the batch on which adversarial samples are generated.
+        :param verbose: Show progress bars.
         """
         super().__init__(
             estimator=estimator,
@@ -232,6 +238,7 @@ def __init__(
             num_random_init=num_random_init,
             batch_size=batch_size,
             random_eps=random_eps,
+            verbose=verbose,
         )
 
         self._project = True
@@ -266,10 +273,12 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
             adv_x_best = None
             rate_best = None
 
-            for _ in range(max(1, self.num_random_init)):
+            for _ in trange(
+                max(1, self.num_random_init), desc="PGD - Random Initializations", disable=not self.verbose
+            ):
                 adv_x = x.astype(ART_NUMPY_DTYPE)
 
-                for i_max_iter in range(self.max_iter):
+                for i_max_iter in trange(self.max_iter, desc="PGD - Iterations", leave=False, disable=not self.verbose):
                     adv_x = self._compute(
                         adv_x,
                         x,
@@ -313,7 +322,7 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
             # Start to compute adversarial examples
             adv_x = x.astype(ART_NUMPY_DTYPE)
 
-            for i_max_iter in range(self.max_iter):
+            for i_max_iter in trange(self.max_iter, desc="PGD - Iterations", disable=not self.verbose):
                 adv_x = self._compute(
                     adv_x,
                     x,

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_pytorch.py

@@ -29,6 +29,7 @@
 from typing import Optional, Union, TYPE_CHECKING
 
 import numpy as np
+from tqdm import trange, tqdm
 
 from art.config import ART_NUMPY_DTYPE
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_numpy import (
@@ -64,6 +65,7 @@ def __init__(
         num_random_init: int = 0,
         batch_size: int = 32,
         random_eps: bool = False,
+        verbose: bool = True,
     ):
         """
         Create a :class:`.ProjectedGradientDescentPytorch` instance.
@@ -81,6 +83,7 @@ def __init__(
         :param num_random_init: Number of random initialisations within the epsilon ball. For num_random_init=0 starting
                                 at the original input.
         :param batch_size: Size of the batch on which adversarial samples are generated.
+        :param verbose: Show progress bars.
         """
         if (
             hasattr(estimator, "preprocessing")
@@ -104,6 +107,7 @@ def __init__(
             num_random_init=num_random_init,
             batch_size=batch_size,
             random_eps=random_eps,
+            verbose=verbose,
         )
 
     def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
@@ -163,11 +167,13 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         adv_x_best = None
         rate_best = None
 
-        for _ in range(max(1, self.num_random_init)):
+        for _ in trange(max(1, self.num_random_init), desc="PGD - Random Initializations", disable=not self.verbose):
             adv_x = x.astype(ART_NUMPY_DTYPE)
 
             # Compute perturbation with batching
-            for (batch_id, batch_all) in enumerate(data_loader):
+            for (batch_id, batch_all) in enumerate(
+                tqdm(data_loader, desc="PGD - Iterations", leave=False, disable=not self.verbose)
+            ):
                 if mask is not None:
                     (batch, batch_labels, mask_batch) = batch_all[0], batch_all[1], batch_all[2]
                 else:

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent_tensorflow_v2.py

@@ -29,6 +29,7 @@
 from typing import Optional, Union, TYPE_CHECKING
 
 import numpy as np
+from tqdm import trange, tqdm
 
 from art.config import ART_NUMPY_DTYPE
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_numpy import (
@@ -63,6 +64,7 @@ def __init__(
         num_random_init: int = 0,
         batch_size: int = 32,
         random_eps: bool = False,
+        verbose: bool = True,
     ):
         """
         Create a :class:`.ProjectedGradientDescentTensorFlowV2` instance.
@@ -80,6 +82,7 @@ def __init__(
         :param num_random_init: Number of random initialisations within the epsilon ball. For num_random_init=0 starting
                                 at the original input.
         :param batch_size: Size of the batch on which adversarial samples are generated.
+        :param verbose: Show progress bars.
         """
         if (
             hasattr(estimator, "preprocessing")
@@ -103,6 +106,7 @@ def __init__(
             num_random_init=num_random_init,
             batch_size=batch_size,
             random_eps=random_eps,
+            verbose=verbose,
         )
 
     def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
@@ -158,12 +162,14 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         adv_x_best = None
         rate_best = None
 
-        for _ in range(max(1, self.num_random_init)):
+        for _ in trange(max(1, self.num_random_init), desc="PGD - Random Initializations", disable=not self.verbose):
             adv_x = x.astype(ART_NUMPY_DTYPE)
             data_loader = iter(dataset)
 
             # Compute perturbation with batching
-            for (batch_id, batch_all) in enumerate(data_loader):
+            for (batch_id, batch_all) in enumerate(
+                tqdm(data_loader, desc="PGD - Iterations", leave=False, disable=not self.verbose)
+            ):
                 if mask is not None:
                     (batch, batch_labels, mask_batch) = batch_all[0], batch_all[1], batch_all[2]
                 else:

art/defences/trainer/adversarial_trainer.py

@@ -120,7 +120,8 @@ def fit_generator(self, generator: "DataGenerator", nb_epochs: int = 20, **kwarg
         # Precompute adversarial samples for transferred attacks
         logged = False
         self._precomputed_adv_samples = []
-        for attack in tqdm(self.attacks, desc="Precompute adv samples"):
+        for attack in tqdm(self.attacks, desc="Precompute adversarial examples."):
+            attack.set_params(verbose=False)
             if "targeted" in attack.attack_params and attack.targeted:  # type: ignore
                 raise NotImplementedError("Adversarial training with targeted attacks is currently not implemented")
 
@@ -154,6 +155,7 @@ def fit_generator(self, generator: "DataGenerator", nb_epochs: int = 20, **kwarg
                 # Choose indices to replace with adversarial samples
                 nb_adv = int(np.ceil(self.ratio * x_batch.shape[0]))
                 attack = self.attacks[attack_id]
+                attack.set_params(verbose=False)
                 if self.ratio < 1:
                     adv_ids = np.random.choice(x_batch.shape[0], size=nb_adv, replace=False)
                 else:
@@ -194,6 +196,7 @@ def fit(self, x: np.ndarray, y: np.ndarray, batch_size: int = 128, nb_epochs: in
         logged = False
         self._precomputed_adv_samples = []
         for attack in tqdm(self.attacks, desc="Precompute adv samples"):
+            attack.set_params(verbose=False)
             if "targeted" in attack.attack_params and attack.targeted:  # type: ignore
                 raise NotImplementedError("Adversarial training with targeted attacks is currently not implemented")
 
@@ -217,6 +220,7 @@ def fit(self, x: np.ndarray, y: np.ndarray, batch_size: int = 128, nb_epochs: in
                 # Choose indices to replace with adversarial samples
                 nb_adv = int(np.ceil(self.ratio * x_batch.shape[0]))
                 attack = self.attacks[attack_id]
+                attack.set_params(verbose=False)
                 if self.ratio < 1:
                     adv_ids = np.random.choice(x_batch.shape[0], size=nb_adv, replace=False)
                 else:

art/defences/trainer/adversarial_trainer_fbf_pytorch.py

@@ -27,6 +27,7 @@
 from typing import Optional, Tuple, Union, TYPE_CHECKING
 
 import numpy as np
+from tqdm import trange
 
 from art.config import ART_NUMPY_DTYPE
 from art.defences.trainer.adversarial_trainer_fbf import AdversarialTrainerFBF
@@ -92,9 +93,9 @@ def fit(
         def lr_schedule(t):
             return np.interp([t], [0, nb_epochs * 2 // 5, nb_epochs], [0, 0.21, 0])[0]
 
-        logger.info("Adversarial training FBF")
+        logger.info("Adversarial Training FBF")
 
-        for i_epoch in range(nb_epochs):
+        for i_epoch in trange(nb_epochs, desc="Adversarial Training FBF - Epochs"):
             # Shuffle the examples
             np.random.shuffle(ind)
             start_time = time.time()
@@ -160,9 +161,9 @@ def fit_generator(self, generator: "DataGenerator", nb_epochs: int = 20, **kwarg
         def lr_schedule(t):
             return np.interp([t], [0, nb_epochs * 2 // 5, nb_epochs], [0, 0.21, 0])[0]
 
-        logger.info("Adversarial training FBF")
+        logger.info("Adversarial Training FBF")
 
-        for i_epoch in range(nb_epochs):
+        for i_epoch in trange(nb_epochs, desc="Adversarial Training FBF - Epochs"):
             start_time = time.time()
             train_loss = 0.0
             train_acc = 0.0