Skip to content

Commit a405f49

Browse files
beat-buesserbeat-buesser
authored
Merge branch 'dev_1.17.0' into simple_pb_inclusion
2 parents 1f3026b + a281f62 commit a405f49

File tree

8 files changed

+1528
-156
lines changed

8 files changed

+1528
-156
lines changed

‎art/attacks/evasion/__init__.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
from art.attacks.evasion.brendel_bethge import BrendelBethgeAttack
1919

2020
from art.attacks.evasion.boundary import BoundaryAttack
21+
from art.attacks.evasion.composite_adversarial_attack import CompositeAdversarialAttackPyTorch
2122
from art.attacks.evasion.carlini import CarliniL2Method, CarliniLInfMethod, CarliniL0Method
2223
from art.attacks.evasion.decision_tree_attack import DecisionTreeAttack
2324
from art.attacks.evasion.deepfool import DeepFool

‎art/attacks/evasion/composite_adversarial_attack.py‎

Lines changed: 673 additions & 0 deletions
Large diffs are not rendered by default.

‎art/attacks/inference/membership_inference/black_box.py‎

Lines changed: 249 additions & 123 deletions
Large diffs are not rendered by default.

‎notebooks/README.md‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,11 @@ demonstrates a MembershipInferenceBlackBox membership inference attack using sha
108108
[label_only_membership_inference.ipynb](label_only_membership_inference.ipynb) [[on nbviewer](https://nbviewer.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/label_only_membership_inference.ipynb)]
109109
demonstrates a LabelOnlyDecisionBoundary membership inference attack on a PyTorch classifier for the MNIST dataset.
110110

111+
[composite-adversarial-attack.ipynb](composite-adversarial-attack.ipynb)[[on nbviewer](https://nbviewer.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/composite-adversarial-attack.ipynb)]
112+
shows how to launch Composite Adversarial Attack (CAA) on Pytorch-based model ([Hsiung et al., 2023](https://arxiv.org/abs/2202.04235)).
113+
CAA composites the perturbations in Lp-ball and semantic space (i.e., hue, saturation, rotation, brightness, and contrast),
114+
and is able to optimize the attack sequence and each attack component, thereby enhancing the efficiency and efficacy of adversarial examples.
115+
111116
## Metrics
112117

113118
[privacy_metric.ipynb](privacy_metric.ipynb) [[on nbviewer](https://nbviewer.jupyter.org/github/Trusted-AI/adversarial-robustness-toolbox/blob/main/notebooks/privacy_metric.ipynb)]

‎notebooks/composite-adversarial-attack.ipynb‎

Lines changed: 290 additions & 0 deletions
Large diffs are not rendered by default.

‎tests/attacks/evasion/test_composite_adversarial_attack.py‎

Lines changed: 192 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,192 @@
1+
# MIT License
2+
#
3+
# Copyright (C) The Adversarial Robustness Toolbox (ART) Authors 2023
4+
#
5+
# Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated
6+
# documentation files (the "Software"), to deal in the Software without restriction, including without limitation the
7+
# rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit
8+
# persons to whom the Software is furnished to do so, subject to the following conditions:
9+
#
10+
# The above copyright notice and this permission notice shall be included in all copies or substantial portions of the
11+
# Software.
12+
#
13+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
14+
# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
15+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
16+
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
17+
# SOFTWARE.
18+
import logging
19+
20+
import numpy as np
21+
import pytest
22+
23+
from art.attacks.evasion import CompositeAdversarialAttackPyTorch
24+
from art.estimators.estimator import BaseEstimator, LossGradientsMixin
25+
from art.estimators.classification.classifier import ClassifierMixin
26+
27+
from tests.attacks.utils import backend_test_classifier_type_check_fail
28+
from tests.utils import ARTTestException, get_cifar10_image_classifier_pt
29+
30+
logger = logging.getLogger(__name__)
31+
32+
33+
@pytest.fixture()
34+
def fix_get_cifar10_subset(get_cifar10_dataset):
35+
(x_train_cifar10, y_train_cifar10), (x_test_cifar10, y_test_cifar10) = get_cifar10_dataset
36+
n_train = 100
37+
n_test = 11
38+
yield x_train_cifar10[:n_train], y_train_cifar10[:n_train], x_test_cifar10[:n_test], y_test_cifar10[:n_test]
39+
40+
41+
@pytest.mark.skip_framework(
42+
"tensorflow1", "tensorflow2", "tensorflow2v1", "keras", "non_dl_frameworks", "mxnet", "kerastf", "huggingface"
43+
)
44+
def test_generate(art_warning, fix_get_cifar10_subset):
45+
try:
46+
(x_train, y_train, x_test, y_test) = fix_get_cifar10_subset
47+
48+
classifier = get_cifar10_image_classifier_pt(from_logits=False, load_init=True)
49+
attack = CompositeAdversarialAttackPyTorch(classifier)
50+
51+
x_train_adv = attack.generate(x=x_train, y=y_train)
52+
x_test_adv = attack.generate(x=x_test, y=y_test)
53+
54+
assert x_train.shape == x_train_adv.shape
55+
assert np.min(x_train_adv) >= 0.0
56+
assert np.max(x_train_adv) <= 1.0
57+
assert x_test.shape == x_test_adv.shape
58+
assert np.min(x_test_adv) >= 0.0
59+
assert np.max(x_test_adv) <= 1.0
60+
61+
except ARTTestException as e:
62+
art_warning(e)
63+
64+
65+
@pytest.mark.skip_framework(
66+
"tensorflow1", "tensorflow2", "tensorflow2v1", "keras", "non_dl_frameworks", "mxnet", "kerastf"
67+
)
68+
def test_check_params(art_warning):
69+
try:
70+
classifier = get_cifar10_image_classifier_pt(from_logits=False, load_init=True)
71+
72+
with pytest.raises(ValueError):
73+
_ = CompositeAdversarialAttackPyTorch(classifier, enabled_attack=(0, 1, 2, 3, 4, 5, 6, 7))
74+
75+
with pytest.raises(ValueError):
76+
_ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(-10.0, 0.0))
77+
with pytest.raises(ValueError):
78+
_ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(0.0, 10.0))
79+
with pytest.raises(TypeError):
80+
_ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(-1, 2.0))
81+
with pytest.raises(TypeError):
82+
_ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=3.14)
83+
with pytest.raises(TypeError):
84+
_ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=(0.0, 10.0, 20.0))
85+
with pytest.raises(TypeError):
86+
_ = CompositeAdversarialAttackPyTorch(classifier, hue_epsilon=("1.0", 2.0))
87+
88+
with pytest.raises(ValueError):
89+
_ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(-10.0, 0.0))
90+
with pytest.raises(ValueError):
91+
_ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(0.0, -10.0))
92+
with pytest.raises(TypeError):
93+
_ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(1, 2.0))
94+
with pytest.raises(TypeError):
95+
_ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=2.0)
96+
with pytest.raises(TypeError):
97+
_ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=(0.0, 10.0, 20.0))
98+
with pytest.raises(TypeError):
99+
_ = CompositeAdversarialAttackPyTorch(classifier, sat_epsilon=("1.0", 2.0))
100+
101+
with pytest.raises(ValueError):
102+
_ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(-450.0, 359.0))
103+
with pytest.raises(ValueError):
104+
_ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(10.0, -10.0))
105+
with pytest.raises(TypeError):
106+
_ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(1.0, 2))
107+
with pytest.raises(TypeError):
108+
_ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=10)
109+
with pytest.raises(TypeError):
110+
_ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=(0.0, 10.0, 20.0))
111+
with pytest.raises(TypeError):
112+
_ = CompositeAdversarialAttackPyTorch(classifier, rot_epsilon=("10", 20.0))
113+
114+
with pytest.raises(ValueError):
115+
_ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(-10.0, 0.0))
116+
with pytest.raises(ValueError):
117+
_ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(0.0, 10.0))
118+
with pytest.raises(TypeError):
119+
_ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(-1, 1.0))
120+
with pytest.raises(TypeError):
121+
_ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=1.0)
122+
with pytest.raises(TypeError):
123+
_ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=(0.0, 10.0, 20.0))
124+
with pytest.raises(TypeError):
125+
_ = CompositeAdversarialAttackPyTorch(classifier, bri_epsilon=("1.0", 2.0))
126+
127+
with pytest.raises(ValueError):
128+
_ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(-10.0, 10.0))
129+
with pytest.raises(ValueError):
130+
_ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(0.0, -10.0))
131+
with pytest.raises(TypeError):
132+
_ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(1, 2.0))
133+
with pytest.raises(TypeError):
134+
_ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=2.0)
135+
with pytest.raises(TypeError):
136+
_ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=(0.0, 10.0, 20.0))
137+
with pytest.raises(TypeError):
138+
_ = CompositeAdversarialAttackPyTorch(classifier, con_epsilon=("1.0", 2.0))
139+
140+
with pytest.raises(ValueError):
141+
_ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(-0.5, 2.0))
142+
with pytest.raises(ValueError):
143+
_ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(8 / 255, -8 / 255))
144+
with pytest.raises(TypeError):
145+
_ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(-2, 1))
146+
with pytest.raises(TypeError):
147+
_ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=8 / 255)
148+
with pytest.raises(TypeError):
149+
_ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=(0.0, 10.0, 20.0))
150+
with pytest.raises(TypeError):
151+
_ = CompositeAdversarialAttackPyTorch(classifier, pgd_epsilon=("2/255", 3 / 255))
152+
153+
with pytest.raises(TypeError):
154+
_ = CompositeAdversarialAttackPyTorch(classifier, early_stop="true")
155+
with pytest.raises(TypeError):
156+
_ = CompositeAdversarialAttackPyTorch(classifier, early_stop=1)
157+
158+
with pytest.raises(TypeError):
159+
_ = CompositeAdversarialAttackPyTorch(classifier, max_iter="max")
160+
with pytest.raises(ValueError):
161+
_ = CompositeAdversarialAttackPyTorch(classifier, max_iter=-5)
162+
with pytest.raises(TypeError):
163+
_ = CompositeAdversarialAttackPyTorch(classifier, max_iter=2.5)
164+
165+
with pytest.raises(TypeError):
166+
_ = CompositeAdversarialAttackPyTorch(classifier, max_inner_iter="max")
167+
with pytest.raises(ValueError):
168+
_ = CompositeAdversarialAttackPyTorch(classifier, max_inner_iter=-5)
169+
with pytest.raises(TypeError):
170+
_ = CompositeAdversarialAttackPyTorch(classifier, max_inner_iter=2.5)
171+
172+
with pytest.raises(ValueError):
173+
_ = CompositeAdversarialAttackPyTorch(classifier, attack_order="schedule")
174+
175+
with pytest.raises(ValueError):
176+
_ = CompositeAdversarialAttackPyTorch(classifier, batch_size=-1)
177+
178+
with pytest.raises(TypeError):
179+
_ = CompositeAdversarialAttackPyTorch(classifier, verbose="true")
180+
181+
except ARTTestException as e:
182+
art_warning(e)
183+
184+
185+
@pytest.mark.framework_agnostic
186+
def test_classifier_type_check_fail(art_warning):
187+
try:
188+
backend_test_classifier_type_check_fail(
189+
CompositeAdversarialAttackPyTorch, [BaseEstimator, LossGradientsMixin, ClassifierMixin]
190+
)
191+
except ARTTestException as e:
192+
art_warning(e)

‎tests/attacks/inference/attribute_inference/test_true_label_baseline.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -605,8 +605,8 @@ def transform_other_feature(x):
605605
baseline_inferred_test
606606
)
607607

608-
expected_train_acc = {"nn": 0.81, "rf": 0.95, "gb": 0.95, "lr": 0.81, "dt": 0.94, "knn": 0.87, "svm": 0.81}
609-
expected_test_acc = {"nn": 0.88, "rf": 0.79, "gb": 0.8, "lr": 0.88, "dt": 0.74, "knn": 0.86, "svm": 0.88}
608+
expected_train_acc = {"nn": 0.81, "rf": 0.93, "gb": 0.95, "lr": 0.81, "dt": 0.94, "knn": 0.87, "svm": 0.81}
609+
expected_test_acc = {"nn": 0.88, "rf": 0.78, "gb": 0.8, "lr": 0.88, "dt": 0.74, "knn": 0.86, "svm": 0.88}
610610

611611
assert expected_train_acc[model_type] <= baseline_train_acc
612612
assert expected_test_acc[model_type] <= baseline_test_acc

0 commit comments

Comments
 (0)