Merge pull request #1154 from Trusted-AI/development_maintenance_170

beat-buesser · web-flow · commit 182f5982814b · 2021-06-15T14:33:59.000+01:00
General maintenance updates for ART 1.7.0
diff --git a/.gitignore b/.gitignore
@@ -96,9 +96,6 @@ ENV/
 *.jpg
 demo/pics/*
 
-# ignore local config
-*config.ini
-
 # Things TF might pull when testing
 *.gz
 *.npy
diff --git a/art/__init__.py b/art/__init__.py
@@ -7,8 +7,9 @@
 from art import attacks
 from art import defences
 from art import estimators
+from art import evaluations
 from art import metrics
-from art import wrappers
+from art import preprocessing
 
 # Semantic Version
 __version__ = "1.7.0-dev"
diff --git a/art/attacks/evasion/auto_projected_gradient_descent.py b/art/attacks/evasion/auto_projected_gradient_descent.py
@@ -382,6 +382,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
                 raise ValueError("Target labels `y` need to be provided for a targeted attack.")
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size)).astype(np.int32)
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         x_adv = x.astype(ART_NUMPY_DTYPE)
 
         for _ in trange(max(1, self.nb_random_init), desc="AutoPGD - restart", disable=not self.verbose):
diff --git a/art/attacks/evasion/boundary.py b/art/attacks/evasion/boundary.py
@@ -125,6 +125,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         """
         y = check_and_transform_label_format(y, self.estimator.nb_classes, return_one_hot=False)
 
+        if y is not None and self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Get clip_min and clip_max from the classifier or infer them from data
         if self.estimator.clip_values is not None:
             clip_min, clip_max = self.estimator.clip_values
diff --git a/art/attacks/evasion/brendel_bethge.py b/art/attacks/evasion/brendel_bethge.py
@@ -2198,6 +2198,11 @@ def generate(  # pylint: disable=W0221
             logger.info("Using model predictions as correct labels for FGM.")
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))  # type: ignore
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Prediction from the initial adversarial examples if not None
         x_adv_init = kwargs.get("x_adv_init")
 
diff --git a/art/attacks/evasion/carlini.py b/art/attacks/evasion/carlini.py
@@ -252,6 +252,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if y is None:
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Compute perturbation with implicit batching
         nb_batches = int(np.ceil(x_adv.shape[0] / float(self.batch_size)))
         for batch_id in trange(nb_batches, desc="C&W L_2", disable=not self.verbose):
@@ -656,6 +661,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if y is None:
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Compute perturbation with implicit batching
         nb_batches = int(np.ceil(x_adv.shape[0] / float(self.batch_size)))
         for batch_id in trange(nb_batches, desc="C&W L_inf", disable=not self.verbose):
@@ -971,6 +981,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if y is None:
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         if self.mask is None:
             # No initial activation provided, use the full feature set
             activation = np.ones(x.shape)
diff --git a/art/attacks/evasion/deepfool.py b/art/attacks/evasion/deepfool.py
@@ -101,6 +101,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         x_adv = x.astype(ART_NUMPY_DTYPE)
         preds = self.estimator.predict(x, batch_size=self.batch_size)
 
+        if self.estimator.nb_classes == 2 and preds.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         if is_probability(preds[0]):
             logger.warning(
                 "It seems that the attacked model is predicting probabilities. DeepFool expects logits as model output "
diff --git a/art/attacks/evasion/elastic_net.py b/art/attacks/evasion/elastic_net.py
@@ -215,6 +215,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if y is None:
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Compute adversarial examples with implicit batching
         nb_batches = int(np.ceil(x_adv.shape[0] / float(self.batch_size)))
         for batch_id in trange(nb_batches, desc="EAD", disable=not self.verbose):
diff --git a/art/attacks/evasion/feature_adversaries/feature_adversaries_pytorch.py b/art/attacks/evasion/feature_adversaries/feature_adversaries_pytorch.py
@@ -21,7 +21,7 @@
 | Paper link: https://arxiv.org/abs/1511.05122
 """
 import logging
-from typing import TYPE_CHECKING, Optional, Union
+from typing import TYPE_CHECKING, Optional, Tuple, Union
 
 import numpy as np
 from tqdm.auto import trange
@@ -68,7 +68,7 @@ def __init__(
         optimizer: Optional["Optimizer"] = None,
         optimizer_kwargs: Optional[dict] = None,
         lambda_: float = 0.0,
-        layer: Union[int, str] = -1,
+        layer: Union[int, str, Tuple[int, ...], Tuple[str, ...]] = -1,
         max_iter: int = 100,
         batch_size: int = 32,
         step_size: Optional[Union[int, float]] = None,
@@ -93,9 +93,9 @@ def __init__(
         """
         super().__init__(estimator=estimator)
 
+        self.delta = delta
         self.optimizer = optimizer
         self._optimizer_kwargs = {} if optimizer_kwargs is None else optimizer_kwargs
-        self.delta = delta
         self.lambda_ = lambda_
         self.layer = layer if isinstance(layer, tuple) else (layer,)
         self.batch_size = batch_size
diff --git a/art/attacks/evasion/frame_saliency.py b/art/attacks/evasion/frame_saliency.py
@@ -125,6 +125,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         else:
             targets = y
 
+        if self.estimator.nb_classes == 2 and targets.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         nb_samples = x.shape[0]
         nb_frames = x.shape[self.frame_index]
         x_adv = x.astype(ART_NUMPY_DTYPE)
diff --git a/art/attacks/evasion/geometric_decision_based_attack.py b/art/attacks/evasion/geometric_decision_based_attack.py
@@ -170,6 +170,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         """
         y = check_and_transform_label_format(y, self.estimator.nb_classes, return_one_hot=True)
 
+        if y is not None and self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         x_adv = x.copy()
 
         # Assert that, if attack is targeted, y is provided
diff --git a/art/attacks/evasion/hop_skip_jump.py b/art/attacks/evasion/hop_skip_jump.py
@@ -127,6 +127,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
 
         y = check_and_transform_label_format(y, self.estimator.nb_classes)
 
+        if y is not None and self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Check whether users need a stateful attack
         resume = kwargs.get("resume")
 
diff --git a/art/attacks/evasion/newtonfool.py b/art/attacks/evasion/newtonfool.py
@@ -88,6 +88,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         y_pred = self.estimator.predict(x, batch_size=self.batch_size)
         pred_class = np.argmax(y_pred, axis=1)
 
+        if self.estimator.nb_classes == 2 and y_pred.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Compute perturbation with implicit batching
         for batch_id in trange(
             int(np.ceil(x_adv.shape[0] / float(self.batch_size))), desc="NewtonFool", disable=not self.verbose
diff --git a/art/attacks/evasion/pixel_threshold.py b/art/attacks/evasion/pixel_threshold.py
@@ -154,6 +154,10 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
                 raise ValueError("Target labels `y` need to be provided for a targeted attack.")
             y = np.argmax(self.estimator.predict(x), axis=1)
         else:
+            if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+                raise ValueError(
+                    "This attack has not yet been tested for binary classification with a single output classifier."
+                )
             if len(y.shape) > 1:
                 y = np.argmax(y, axis=1)
 
diff --git a/art/attacks/evasion/saliency_map.py b/art/attacks/evasion/saliency_map.py
@@ -98,6 +98,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
 
             targets = np.argmax(random_targets(preds, self.estimator.nb_classes), axis=1)
         else:
+            if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+                raise ValueError(
+                    "This attack has not yet been tested for binary classification with a single output classifier."
+                )
+
             targets = np.argmax(y, axis=1)
 
         # Compute perturbation with implicit batching
diff --git a/art/attacks/evasion/shadow_attack.py b/art/attacks/evasion/shadow_attack.py
@@ -132,6 +132,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         else:
             self.targeted = True
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         if x.shape[0] > 1 or y.shape[0] > 1:
             raise ValueError("This attack only accepts a single sample as input.")
 
diff --git a/art/attacks/evasion/simba.py b/art/attacks/evasion/simba.py
@@ -107,6 +107,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         x = x.astype(ART_NUMPY_DTYPE)
         preds = self.estimator.predict(x, batch_size=self.batch_size)
 
+        if self.estimator.nb_classes == 2 and preds.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         if divmod(x.shape[2] - self.freq_dim, self.stride)[1] != 0:
             raise ValueError(
                 "Incompatible value combination in image height/width, freq_dim and stride detected. "
diff --git a/art/attacks/evasion/spatial_transformation.py b/art/attacks/evasion/spatial_transformation.py
@@ -108,6 +108,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if self.attack_trans_x is None or self.attack_trans_y is None or self.attack_rot is None:
 
             y_pred = self.estimator.predict(x, batch_size=1)
+            if self.estimator.nb_classes == 2 and y_pred.shape[1] == 1:
+                raise ValueError(
+                    "This attack has not yet been tested for binary classification with a single output classifier."
+                )
+
             y_pred_max = np.argmax(y_pred, axis=1)
 
             nb_instances = len(x)
diff --git a/art/attacks/evasion/square_attack.py b/art/attacks/evasion/square_attack.py
@@ -159,6 +159,12 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
             if isinstance(self.estimator, ClassifierMixin):
                 y = get_labels_np_array(y)
 
+        if isinstance(self.estimator, ClassifierMixin):
+            if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+                raise ValueError(
+                    "This attack has not yet been tested for binary classification with a single output classifier."
+                )
+
         if self.estimator.channels_first:
             channels = x.shape[1]
             height = x.shape[2]
diff --git a/art/attacks/evasion/targeted_universal_perturbation.py b/art/attacks/evasion/targeted_universal_perturbation.py
@@ -98,6 +98,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if y is None:
             raise ValueError("Labels `y` cannot be None.")
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         logger.info("Computing targeted universal perturbation based on %s attack.", self.attacker)
 
         # Init universal perturbation
diff --git a/art/attacks/evasion/universal_perturbation.py b/art/attacks/evasion/universal_perturbation.py
@@ -160,6 +160,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
             logger.info("Using model predictions as true labels.")
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         y_index = np.argmax(y, axis=1)
 
         # Init universal perturbation
diff --git a/art/attacks/evasion/virtual_adversarial.py b/art/attacks/evasion/virtual_adversarial.py
@@ -93,6 +93,12 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         """
         x_adv = x.astype(ART_NUMPY_DTYPE)
         preds = self.estimator.predict(x_adv, batch_size=self.batch_size)
+
+        if self.estimator.nb_classes == 2 and preds.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         if (preds < 0.0).any() or (preds > 1.0).any():
             raise TypeError(
                 "This attack requires a classifier predicting probabilities in the range [0, 1] as output."
diff --git a/art/attacks/evasion/wasserstein.py b/art/attacks/evasion/wasserstein.py
@@ -154,6 +154,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         else:
             targets = y
 
+        if self.estimator.nb_classes == 2 and targets.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Compute the cost matrix if needed
         cost_matrix = kwargs.get("cost_matrix")
         if cost_matrix is None:
diff --git a/art/attacks/evasion/zoo.py b/art/attacks/evasion/zoo.py
@@ -218,6 +218,11 @@ def generate(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> n
         if y is None:
             y = get_labels_np_array(self.estimator.predict(x, batch_size=self.batch_size))
 
+        if self.estimator.nb_classes == 2 and y.shape[1] == 1:
+            raise ValueError(
+                "This attack has not yet been tested for binary classification with a single output classifier."
+            )
+
         # Compute adversarial examples with implicit batching
         nb_batches = int(np.ceil(x.shape[0] / float(self.batch_size)))
         x_adv = []
diff --git a/art/classifiers/__init__.py b/art/classifiers/__init__.py
@@ -2,19 +2,29 @@
 Classifier API for applying all attacks. Use the :class:`.Classifier` wrapper to be able to apply an attack to a
 existing model.
 """
-from art.estimators.classification.blackbox import BlackBoxClassifier
-from art.estimators.classification.catboost import CatBoostARTClassifier
-from art.estimators.classification.detector_classifier import DetectorClassifier
-from art.estimators.classification.ensemble import EnsembleClassifier
-from art.estimators.classification.GPy import GPyGaussianProcessClassifier
-from art.estimators.classification.keras import KerasClassifier
-from art.estimators.classification.lightgbm import LightGBMClassifier
-from art.estimators.classification.mxnet import MXClassifier
-from art.estimators.classification.pytorch import PyTorchClassifier
-from art.estimators.classification.scikitlearn import SklearnClassifier
-from art.estimators.classification.tensorflow import (
+# pylint: disable=C0413
+import warnings
+
+warnings.simplefilter("always", category=DeprecationWarning)
+warnings.warn(
+    "The module art.classifiers will be removed in ART 1.8.0 and replaced with art.estimators.classification",
+    DeprecationWarning,
+    stacklevel=2,
+)
+warnings.simplefilter("default", category=DeprecationWarning)
+from art.estimators.classification.blackbox import BlackBoxClassifier  # noqa
+from art.estimators.classification.catboost import CatBoostARTClassifier  # noqa
+from art.estimators.classification.detector_classifier import DetectorClassifier  # noqa
+from art.estimators.classification.ensemble import EnsembleClassifier  # noqa
+from art.estimators.classification.GPy import GPyGaussianProcessClassifier  # noqa
+from art.estimators.classification.keras import KerasClassifier  # noqa
+from art.estimators.classification.lightgbm import LightGBMClassifier  # noqa
+from art.estimators.classification.mxnet import MXClassifier  # noqa
+from art.estimators.classification.pytorch import PyTorchClassifier  # noqa
+from art.estimators.classification.scikitlearn import SklearnClassifier  # noqa
+from art.estimators.classification.tensorflow import (  # noqa
     TFClassifier,
     TensorFlowClassifier,
     TensorFlowV2Classifier,
 )
-from art.estimators.classification.xgboost import XGBoostClassifier
+from art.estimators.classification.xgboost import XGBoostClassifier  # noqa
diff --git a/art/estimators/classification/pytorch.py b/art/estimators/classification/pytorch.py
diff --git a/art/estimators/classification/tensorflow.py b/art/estimators/classification/tensorflow.py
diff --git a/art/wrappers/__init__.py b/art/wrappers/__init__.py
diff --git a/art/wrappers/expectation.py b/art/wrappers/expectation.py
diff --git a/art/wrappers/query_efficient_bb.py b/art/wrappers/query_efficient_bb.py
diff --git a/conftest.py b/conftest.py
diff --git a/docs/conf.py b/docs/conf.py
diff --git a/docs/modules/attacks/evasion.rst b/docs/modules/attacks/evasion.rst
diff --git a/docs/modules/attacks/inference/attribute_inference.rst b/docs/modules/attacks/inference/attribute_inference.rst
diff --git a/docs/modules/estimators/speech_recognition.rst b/docs/modules/estimators/speech_recognition.rst
diff --git a/setup.py b/setup.py
