Merge remote-tracking branch 'origin/dev_1.8.0' into development_documentation

Author: Beat Buesser

.github/workflows/ci-pytorch-fasterrcnn.yml renamed to .github/workflows/ci-pytorch-object-detectors.yml

@@ -1,4 +1,4 @@
-name: CI PyTorchFasterRCNN
+name: CI PyTorchObjectDetectors
 on:
   # Run on manual trigger
   workflow_dispatch:
@@ -20,7 +20,7 @@ on:
 
 jobs:
   test_pytorch_fasterrcnn:
-    name: PyTorchFasterRCNN
+    name: PyTorchObjectDetectors
     runs-on: ubuntu-20.04
     strategy:
       fail-fast: false
@@ -43,7 +43,9 @@ jobs:
           pip install torch==1.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
           pip install torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
           pip install torchaudio==0.6.0 -f https://download.pytorch.org/whl/torch_stable.html
-      - name: Run Test Action
+      - name: Run Test Action - test_pytorch_object_detector
+        run: pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/object_detection/test_pytorch_object_detector.py --framework=pytorch --durations=0
+      - name: Run Test Action - test_pytorch_faster_rcnn
         run: pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/object_detection/test_pytorch_faster_rcnn.py --framework=pytorch --durations=0
       - name: Upload coverage to Codecov
         uses: codecov/[email protected]

.github/workflows/ci-style-checks.yml

@@ -51,7 +51,7 @@ jobs:
         run: mypy art
       - name: pytest-flake8
         if: ${{ always() }}
-        run: pytest --flake8 -v -m flake8
+        run: pytest --flake8 -v -m flake8 --ignore=contrib
       - name: black
         if: ${{ always() }}
         run: |

art/__init__.py

@@ -12,7 +12,7 @@
 from art import preprocessing
 
 # Semantic Version
-__version__ = "1.7.2"
+__version__ = "1.8.0.dev0"
 
 # pylint: disable=C0103
 

art/attacks/attack.py

@@ -91,6 +91,10 @@ class Attack(abc.ABC):
     """
 
     attack_params: List[str] = list()
+    # The _estimator_requirements define the requirements an estimator must satisfy to be used as a target for an
+    # attack. They should be a tuple of requirements, where each requirement is either a class the estimator must
+    # inherit from, or a tuple of classes which define a union, i.e. the estimator must inherit from at least one class
+    # in the requirement tuple.
     _estimator_requirements: Optional[Union[Tuple[Any, ...], Tuple[()]]] = None
 
     def __init__(
@@ -111,7 +115,7 @@ def __init__(
         if self.estimator_requirements is None:
             raise ValueError("Estimator requirements have not been defined in `_estimator_requirements`.")
 
-        if not all(t in type(estimator).__mro__ for t in self.estimator_requirements):
+        if not self.is_estimator_valid(estimator):
             raise EstimatorError(self.__class__, self.estimator_requirements, estimator)
 
         self._estimator = estimator
@@ -155,6 +159,24 @@ def _check_params(self) -> None:
         if not isinstance(self.tensor_board, (bool, str)):
             raise ValueError("The argument `tensor_board` has to be either of type bool or str.")
 
+    def is_estimator_valid(self, estimator) -> bool:
+        """
+        Checks if the given estimator satisfies the requirements for this attack.
+
+        :param estimator: The estimator to check.
+        :return: True if the estimator is valid for the attack.
+        """
+
+        for req in self.estimator_requirements:
+            # A requirement is either a class which the estimator must inherit from, or a tuple of classes and the
+            # estimator is required to inherit from at least one of the classes
+            if isinstance(req, tuple):
+                if all(p not in type(estimator).__mro__ for p in req):
+                    return False
+            elif req not in type(estimator).__mro__:
+                return False
+        return True
+
 
 class EvasionAttack(Attack):
     """
@@ -373,7 +395,7 @@ class MembershipInferenceAttack(InferenceAttack):
     Abstract base class for membership inference attack classes.
     """
 
-    def __init__(self, estimator: Union["CLASSIFIER_TYPE"]):
+    def __init__(self, estimator):
         """
         :param estimator: A trained estimator targeted for inference attack.
         :type estimator: :class:`.art.estimators.estimator.BaseEstimator`

art/attacks/evasion/projected_gradient_descent/projected_gradient_descent.py

@@ -45,7 +45,7 @@
 )
 
 if TYPE_CHECKING:
-    from art.utils import CLASSIFIER_LOSS_GRADIENTS_TYPE
+    from art.utils import CLASSIFIER_LOSS_GRADIENTS_TYPE, OBJECT_DETECTOR_TYPE
 
 logger = logging.getLogger(__name__)
 
@@ -76,7 +76,7 @@ class ProjectedGradientDescent(EvasionAttack):
 
     def __init__(
         self,
-        estimator: "CLASSIFIER_LOSS_GRADIENTS_TYPE",
+        estimator: Union["CLASSIFIER_LOSS_GRADIENTS_TYPE", "OBJECT_DETECTOR_TYPE"],
         norm: Union[int, float, str] = np.inf,
         eps: Union[int, float, np.ndarray] = 0.3,
         eps_step: Union[int, float, np.ndarray] = 0.1,

art/attacks/inference/attribute_inference/__init__.py

@@ -3,6 +3,7 @@
 """
 from art.attacks.inference.attribute_inference.black_box import AttributeInferenceBlackBox
 from art.attacks.inference.attribute_inference.baseline import AttributeInferenceBaseline
+from art.attacks.inference.attribute_inference.true_label_baseline import AttributeInferenceBaselineTrueLabel
 from art.attacks.inference.attribute_inference.white_box_decision_tree import AttributeInferenceWhiteBoxDecisionTree
 from art.attacks.inference.attribute_inference.white_box_lifestyle_decision_tree import (
     AttributeInferenceWhiteBoxLifestyleDecisionTree,

art/attacks/inference/attribute_inference/black_box.py

@@ -29,10 +29,11 @@
 from art.estimators.estimator import BaseEstimator
 from art.estimators.classification.classifier import ClassifierMixin
 from art.attacks.attack import AttributeInferenceAttack
+from art.estimators.regression import RegressorMixin
 from art.utils import check_and_transform_label_format, float_to_categorical, floats_to_one_hot
 
 if TYPE_CHECKING:
-    from art.utils import CLASSIFIER_TYPE
+    from art.utils import CLASSIFIER_TYPE, REGRESSOR_TYPE
 
 logger = logging.getLogger(__name__)
 
@@ -47,23 +48,27 @@ class AttributeInferenceBlackBox(AttributeInferenceAttack):
     used as a proxy.
     """
 
-    _estimator_requirements = (BaseEstimator, ClassifierMixin)
+    attack_params = AttributeInferenceAttack.attack_params + ["prediction_normal_factor"]
+    _estimator_requirements = (BaseEstimator, (ClassifierMixin, RegressorMixin))
 
     def __init__(
         self,
-        classifier: "CLASSIFIER_TYPE",
+        estimator: Union["CLASSIFIER_TYPE", "REGRESSOR_TYPE"],
         attack_model: Optional["CLASSIFIER_TYPE"] = None,
         attack_feature: Union[int, slice] = 0,
+        prediction_normal_factor: float = 1,
     ):
         """
         Create an AttributeInferenceBlackBox attack instance.
 
-        :param classifier: Target classifier.
+        :param estimator: Target estimator.
         :param attack_model: The attack model to train, optional. If none is provided, a default model will be created.
         :param attack_feature: The index of the feature to be attacked or a slice representing multiple indexes in
                                case of a one-hot encoded feature.
+        :param prediction_normal_factor: If supplied, predictions of the model are multiplied by the factor when used as
+                                         inputs to the attack-model. Only applicable when `estimator` is a regressor.
         """
-        super().__init__(estimator=classifier, attack_feature=attack_feature)
+        super().__init__(estimator=estimator, attack_feature=attack_feature)
         if isinstance(self.attack_feature, int):
             self.single_index_feature = True
         else:
@@ -99,6 +104,9 @@ def __init__(
                 n_iter_no_change=10,
                 max_fun=15000,
             )
+
+        self.prediction_normal_factor = prediction_normal_factor
+
         self._check_params()
 
     def fit(self, x: np.ndarray) -> None:
@@ -111,12 +119,15 @@ def fit(self, x: np.ndarray) -> None:
         # Checks:
         if self.estimator.input_shape is not None:
             if self.estimator.input_shape[0] != x.shape[1]:
-                raise ValueError("Shape of x does not match input_shape of classifier")
+                raise ValueError("Shape of x does not match input_shape of model")
         if self.single_index_feature and self.attack_feature >= x.shape[1]:
             raise ValueError("attack_feature must be a valid index to a feature in x")
 
         # get model's predictions for x
-        predictions = np.array([np.argmax(arr) for arr in self.estimator.predict(x)]).reshape(-1, 1)
+        if ClassifierMixin in type(self.estimator).__mro__:
+            predictions = np.array([np.argmax(arr) for arr in self.estimator.predict(x)]).reshape(-1, 1)
+        else:  # Regression model
+            predictions = self.estimator.predict(x).reshape(-1, 1) * self.prediction_normal_factor
 
         # get vector of attacked feature
         y = x[:, self.attack_feature]
@@ -155,9 +166,12 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
             raise ValueError("Number of rows in x and y do not match")
         if self.estimator.input_shape is not None:
             if self.single_index_feature and self.estimator.input_shape[0] != x.shape[1] + 1:
-                raise ValueError("Number of features in x + 1 does not match input_shape of classifier")
+                raise ValueError("Number of features in x + 1 does not match input_shape of model")
 
-        x_test = np.concatenate((x, y), axis=1).astype(np.float32)
+        if RegressorMixin in type(self.estimator).__mro__:
+            x_test = np.concatenate((x, y * self.prediction_normal_factor), axis=1).astype(np.float32)
+        else:
+            x_test = np.concatenate((x, y), axis=1).astype(np.float32)
 
         if self.single_index_feature:
             if "values" not in kwargs.keys():
@@ -182,3 +196,7 @@ def _check_params(self) -> None:
             raise ValueError("Attack feature must be either an integer or a slice object.")
         if isinstance(self.attack_feature, int) and self.attack_feature < 0:
             raise ValueError("Attack feature index must be positive.")
+
+        if RegressorMixin not in type(self.estimator).__mro__:
+            if self.prediction_normal_factor != 1:
+                raise ValueError("Prediction normal factor is only applicable to regressor models.")

art/attacks/inference/attribute_inference/meminf_based.py

@@ -28,10 +28,11 @@
 from art.estimators.estimator import BaseEstimator
 from art.estimators.classification.classifier import ClassifierMixin
 from art.attacks.attack import AttributeInferenceAttack, MembershipInferenceAttack
+from art.estimators.regression import RegressorMixin
 from art.exceptions import EstimatorError
 
 if TYPE_CHECKING:
-    from art.utils import CLASSIFIER_TYPE
+    from art.utils import CLASSIFIER_TYPE, REGRESSOR_TYPE
 
 logger = logging.getLogger(__name__)
 
@@ -44,26 +45,26 @@ class AttributeInferenceMembership(AttributeInferenceAttack):
     as a member with the highest confidence.
     """
 
-    _estimator_requirements = (BaseEstimator, ClassifierMixin)
+    _estimator_requirements = (BaseEstimator, (ClassifierMixin, RegressorMixin))
 
     def __init__(
         self,
-        classifier: "CLASSIFIER_TYPE",
+        estimator: Union["CLASSIFIER_TYPE", "REGRESSOR_TYPE"],
         membership_attack: MembershipInferenceAttack,
         attack_feature: Union[int, slice] = 0,
     ):
         """
         Create an AttributeInferenceMembership attack instance.
 
-        :param classifier: Target classifier.
+        :param estimator: Target estimator.
         :param membership_attack: The membership inference attack to use. Should be fit/calibrated in advance, and
-                                  should support returning probabilities.
+                                  should support returning probabilities. Should also support the target estimator.
         :param attack_feature: The index of the feature to be attacked or a slice representing multiple indexes in
                                case of a one-hot encoded feature.
         """
-        super().__init__(estimator=classifier, attack_feature=attack_feature)
-        if not all(t in type(classifier).__mro__ for t in membership_attack.estimator_requirements):
-            raise EstimatorError(membership_attack, membership_attack.estimator_requirements, classifier)
+        super().__init__(estimator=estimator, attack_feature=attack_feature)
+        if not membership_attack.is_estimator_valid(estimator):
+            raise EstimatorError(membership_attack.__class__, membership_attack.estimator_requirements, estimator)
 
         self.membership_attack = membership_attack
         self._check_params()
@@ -84,7 +85,7 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
         """
         if self.estimator.input_shape is not None:
             if isinstance(self.attack_feature, int) and self.estimator.input_shape[0] != x.shape[1] + 1:
-                raise ValueError("Number of features in x + 1 does not match input_shape of classifier")
+                raise ValueError("Number of features in x + 1 does not match input_shape of the estimator")
 
         if "values" not in kwargs.keys():
             raise ValueError("Missing parameter `values`.")
@@ -96,11 +97,11 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
             if y.shape[0] != x.shape[0]:
                 raise ValueError("Number of rows in x and y do not match")
 
-        # assumes single index
+        # single index
         if isinstance(self.attack_feature, int):
             first = True
             for value in values:
-                v_full = np.full((x.shape[0], 1), value).astype(np.float32)
+                v_full = np.full((x.shape[0], 1), value).astype(x.dtype)
                 x_value = np.concatenate((x[:, : self.attack_feature], v_full), axis=1)
                 x_value = np.concatenate((x_value, x[:, self.attack_feature :]), axis=1)
 
@@ -112,7 +113,7 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
                     probabilities = np.hstack((probabilities, predicted))
 
             # needs to be of type float so we can later replace back the actual values
-            value_indexes = np.argmax(probabilities, axis=1).astype(np.float32)
+            value_indexes = np.argmax(probabilities, axis=1).astype(x.dtype)
             pred_values = np.zeros_like(value_indexes)
             for index, value in enumerate(values):
                 pred_values[value_indexes == index] = value
@@ -134,7 +135,7 @@ def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.n
                 else:
                     probabilities = np.hstack((probabilities, predicted))
                 first = False
-            value_indexes = np.argmax(probabilities, axis=1).astype(np.float32)
+            value_indexes = np.argmax(probabilities, axis=1).astype(x.dtype)
             pred_values = np.zeros_like(probabilities)
             for index, value in enumerate(values):
                 curr_value = np.zeros(len(values))