Merge pull request #1147 from Trusted-AI/dev_1.7.0

Update to ART 1.7.0

Author: beat-buesser

.github/actions/deepspeech-v3/Dockerfile

@@ -0,0 +1,41 @@
+# Get base from a pytorch image
+FROM pytorch/pytorch:1.6.0-cuda10.1-cudnn7-runtime
+
+# Set to install things in non-interactive mode
+ENV DEBIAN_FRONTEND noninteractive
+
+# Install system wide softwares
+RUN apt-get update \
+     && apt-get install -y \
+        libgl1-mesa-glx \
+        libx11-xcb1 \
+        git \
+        gcc \
+        mono-mcs \
+        libavcodec-extra \
+        ffmpeg \
+        curl \
+        libsndfile-dev \
+        libsndfile1 \
+     && apt-get clean all \
+     && rm -r /var/lib/apt/lists/*
+
+RUN /opt/conda/bin/conda install --yes \
+    astropy \
+    matplotlib \
+    pandas \
+    scikit-learn \
+    scikit-image
+
+# Install necessary libraries for deepspeech v3
+RUN pip install torch
+RUN pip install tensorflow
+RUN pip install torchaudio==0.6.0
+RUN pip install --no-build-isolation fairscale
+
+RUN git clone https://github.com/SeanNaren/deepspeech.pytorch.git
+RUN cd deepspeech.pytorch && pip install -r requirements.txt
+RUN cd deepspeech.pytorch && pip install -e .
+
+RUN pip install numba==0.50.0
+RUN pip install pytest-cov

.github/actions/deepspeech-v3/action.yml

@@ -0,0 +1,7 @@
+name: 'Test DeepSpeech v3'
+description: 'Run tests for DeepSpeech v3'
+runs:
+  using: 'composite'
+  steps:
+    - run: $GITHUB_ACTION_PATH/run.sh
+      shell: bash

.github/actions/deepspeech-v3/run.sh

@@ -0,0 +1,10 @@
+#!/bin/sh -l
+
+exit_code=0
+
+pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/speech_recognition/test_pytorch_deep_speech.py --framework=pytorch --skip_travis=True --durations=0
+if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/speech_recognition/test_pytorch_deep_speech tests"; fi
+pytest --cov-report=xml --cov=art --cov-append -q -vv tests/attacks/evasion/test_imperceptible_asr_pytorch.py --framework=pytorch --skip_travis=True --durations=0
+if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion/test_imperceptible_asr_pytorch tests"; fi
+
+exit ${exit_code}

.github/workflows/ci-deepspeech-v3.yml

@@ -0,0 +1,32 @@
+name: CI PyTorchDeepSpeech v3
+on:
+  # Run on manual trigger
+  workflow_dispatch:
+
+  # Run on pull requests
+  pull_request:
+    paths-ignore:
+      - '*.md'
+
+  # Run when pushing to main or dev branches
+  push:
+    branches:
+      - main
+      - dev*
+
+  # Run scheduled CI flow daily
+  schedule:
+    - cron: '0 8 * * 0'
+
+jobs:
+  test_deepspeech_v3:
+    name: PyTorchDeepSpeech v3
+    runs-on: ubuntu-latest
+    container: minhitbk/art_testing_envs:deepspeech_v3
+    steps:
+      - name: Checkout Repo
+        uses: actions/[email protected]
+      - name: Run Test Action
+        uses: ./.github/actions/deepspeech-v3
+      - name: Upload coverage to Codecov
+        uses: codecov/[email protected]

.gitignore

@@ -96,9 +96,6 @@ ENV/
 *.jpg
 demo/pics/*
 
-# ignore local config
-*config.ini
-
 # Things TF might pull when testing
 *.gz
 *.npy

AUTHORS

@@ -10,3 +10,9 @@
 - Intel Corporation
 - University of Chicago
 - The MITRE Corporation
+- General Motors Company
+- AGH University of Science and Technology
+- Rensselaer Polytechnic Institute (RPI)
+- IMT Atlantique
+- Johns Hopkins University
+- Troj.AI

Dockerfile

@@ -31,6 +31,8 @@ RUN pip3 install lightgbm==2.3.1
 RUN pip3 install xgboost==1.1.1
 RUN pip3 install kornia==0.3.1
 
+RUN pip3 install lief==0.11.4
+
 RUN pip3 install pytest==5.4.1 pytest-pep8==1.0.6 pytest-mock==3.2.0 codecov==2.1.8 requests==2.24.0
 
 RUN mkdir /project; mkdir /project/TMP

art/__init__.py

@@ -7,11 +7,12 @@
 from art import attacks
 from art import defences
 from art import estimators
+from art import evaluations
 from art import metrics
-from art import wrappers
+from art import preprocessing
 
 # Semantic Version
-__version__ = "1.6.2"
+__version__ = "1.7.0-dev"
 
 # pylint: disable=C0103
 

art/attacks/attack.py

@@ -93,9 +93,18 @@ class Attack(abc.ABC, metaclass=InputFilter):
     attack_params: List[str] = list()
     _estimator_requirements: Optional[Union[Tuple[Any, ...], Tuple[()]]] = None
 
-    def __init__(self, estimator):
+    def __init__(
+        self,
+        estimator,
+        tensor_board: Union[str, bool] = False,
+    ):
         """
         :param estimator: An estimator.
+        :param tensor_board: Activate summary writer for TensorBoard: Default is `False` and deactivated summary writer.
+                             If `True` save runs/CURRENT_DATETIME_HOSTNAME in current directory. Provide `path` in type
+                             `str` to save in path/CURRENT_DATETIME_HOSTNAME.
+                             Use hierarchical folder structure to compare between runs easily. e.g. pass in ‘runs/exp1’,
+                             ‘runs/exp2’, etc. for each new experiment to compare across them.
         """
         super().__init__()
 
@@ -106,6 +115,19 @@ def __init__(self, estimator):
             raise EstimatorError(self.__class__, self.estimator_requirements, estimator)
 
         self._estimator = estimator
+        self.tensor_board = tensor_board
+
+        if tensor_board:
+            from tensorboardX import SummaryWriter
+
+            if isinstance(tensor_board, str):
+                self.summary_writer = SummaryWriter(tensor_board)
+            else:
+                self.summary_writer = SummaryWriter()
+        else:
+            self.summary_writer = None
+
+        Attack._check_params(self)
 
     @property
     def estimator(self):
@@ -129,7 +151,9 @@ def set_params(self, **kwargs) -> None:
         self._check_params()
 
     def _check_params(self) -> None:
-        pass
+
+        if not isinstance(self.tensor_board, (bool, str)):
+            raise ValueError("The argument `tensor_board` has to be either of type bool or str.")
 
 
 class EvasionAttack(Attack):
@@ -305,12 +329,12 @@ def __init__(self, estimator):
     @abc.abstractmethod
     def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
         """
-        Infer sensitive properties (attributes, membership training records) from the targeted estimator. This method
+        Infer sensitive attributes from the targeted estimator. This method
         should be overridden by all concrete inference attack implementations.
 
         :param x: An array with reference inputs to be used in the attack.
         :param y: Labels for `x`. This parameter is only used by some of the attacks.
-        :return: An array holding the inferred properties.
+        :return: An array holding the inferred attribute values.
         """
         raise NotImplementedError
 
@@ -334,12 +358,41 @@ def __init__(self, estimator, attack_feature: Union[int, slice] = 0):
     @abc.abstractmethod
     def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
         """
-        Infer sensitive properties (attributes, membership training records) from the targeted estimator. This method
+        Infer sensitive attributes from the targeted estimator. This method
+        should be overridden by all concrete inference attack implementations.
+
+        :param x: An array with reference inputs to be used in the attack.
+        :param y: Labels for `x`. This parameter is only used by some of the attacks.
+        :return: An array holding the inferred attribute values.
+        """
+        raise NotImplementedError
+
+
+class MembershipInferenceAttack(InferenceAttack):
+    """
+    Abstract base class for membership inference attack classes.
+    """
+
+    def __init__(self, estimator: Union["CLASSIFIER_TYPE"]):
+        """
+        :param estimator: A trained estimator targeted for inference attack.
+        :type estimator: :class:`.art.estimators.estimator.BaseEstimator`
+        :param attack_feature: The index of the feature to be attacked.
+        """
+        super().__init__(estimator)
+
+    @abc.abstractmethod
+    def infer(self, x: np.ndarray, y: Optional[np.ndarray] = None, **kwargs) -> np.ndarray:
+        """
+        Infer membership status of samples from the target estimator. This method
         should be overridden by all concrete inference attack implementations.
 
         :param x: An array with reference inputs to be used in the attack.
         :param y: Labels for `x`. This parameter is only used by some of the attacks.
-        :return: An array holding the inferred properties.
+        :param probabilities: a boolean indicating whether to return the predicted probabilities per class, or just
+                              the predicted class.
+        :return: An array holding the inferred membership status (1 indicates member of training set,
+                 0 indicates non-member) or class probabilities.
         """
         raise NotImplementedError
 

art/attacks/evasion/__init__.py

@@ -10,21 +10,26 @@
 from art.attacks.evasion.auto_projected_gradient_descent import AutoProjectedGradientDescent
 from art.attacks.evasion.brendel_bethge import BrendelBethgeAttack
 from art.attacks.evasion.boundary import BoundaryAttack
-from art.attacks.evasion.carlini import CarliniL2Method, CarliniLInfMethod
+from art.attacks.evasion.carlini import CarliniL2Method, CarliniLInfMethod, CarliniL0Method
 from art.attacks.evasion.decision_tree_attack import DecisionTreeAttack
 from art.attacks.evasion.deepfool import DeepFool
 from art.attacks.evasion.dpatch import DPatch
 from art.attacks.evasion.dpatch_robust import RobustDPatch
 from art.attacks.evasion.elastic_net import ElasticNet
 from art.attacks.evasion.fast_gradient import FastGradientMethod
 from art.attacks.evasion.frame_saliency import FrameSaliencyAttack
-from art.attacks.evasion.feature_adversaries import FeatureAdversaries
+from art.attacks.evasion.feature_adversaries.feature_adversaries_numpy import FeatureAdversariesNumpy
+from art.attacks.evasion.feature_adversaries.feature_adversaries_pytorch import FeatureAdversariesPyTorch
+from art.attacks.evasion.feature_adversaries.feature_adversaries_tensorflow import FeatureAdversariesTensorFlowV2
+from art.attacks.evasion.geometric_decision_based_attack import GeoDA
 from art.attacks.evasion.hclu import HighConfidenceLowUncertainty
 from art.attacks.evasion.hop_skip_jump import HopSkipJump
 from art.attacks.evasion.imperceptible_asr.imperceptible_asr import ImperceptibleASR
 from art.attacks.evasion.imperceptible_asr.imperceptible_asr_pytorch import ImperceptibleASRPyTorch
 from art.attacks.evasion.iterative_method import BasicIterativeMethod
+from art.attacks.evasion.lowprofool import LowProFool
 from art.attacks.evasion.newtonfool import NewtonFool
+from art.attacks.evasion.pe_malware_attack import MalwareGDTensorFlow
 from art.attacks.evasion.pixel_threshold import PixelAttack
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent import ProjectedGradientDescent
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_numpy import (
@@ -36,6 +41,7 @@
 from art.attacks.evasion.projected_gradient_descent.projected_gradient_descent_tensorflow_v2 import (
     ProjectedGradientDescentTensorFlowV2,
 )
+from art.attacks.evasion.over_the_air_flickering.over_the_air_flickering_pytorch import OverTheAirFlickeringPyTorch
 from art.attacks.evasion.saliency_map import SaliencyMapMethod
 from art.attacks.evasion.shadow_attack import ShadowAttack
 from art.attacks.evasion.shapeshifter import ShapeShifter