Skip to content

Commit 7026c75

Browse files
beat-buesserbeat-buesser
committed
Improve testing setup
Signed-off-by: Beat Buesser <[email protected]>
1 parent c7d5845 commit 7026c75

File tree

3 files changed

+30
-28
lines changed

3 files changed

+30
-28
lines changed

‎pyproject.toml

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,3 +5,10 @@ exclude = [".venv", "contrib"]
55
lint.ignore = ["F401"]
66
[tool.ruff.lint.per-file-ignores]
77
"examples/*" = ["E402"]
8+
[tool.pytest.ini_options]
9+
markers = [
10+
"skip_framework(*frameworks): marks a test to be skipped for specific framework values. Valid values are (\"tensorflow\", \"keras\", \"pytorch\", \"scikitlearn,")"
11+
"only_with_platform(*frameworks): DEPRECATED only used for legacy tests. Use skip_framework instead. marks a test to be performed only for a specific framework value"
12+
"framework_agnostic: marks a test to be agnostic to frameworks and run only for one default framework"
13+
"skip_module(*frameworks): Skip this test for specified frameworks"
14+
]

‎run_tests.sh

Lines changed: 23 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -19,70 +19,70 @@ then
1919
echo "############### Running tests with framework $framework ###############"
2020
echo "#######################################################################"
2121

22-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/detector/evasion --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
22+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/detector/evasion --framework=$framework --durations=20 --durations-min=0 --setup-show
2323
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/detector/evasion tests"; fi
2424

25-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/detector/poison/test_spectral_signature_defense.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
25+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/detector/poison/test_spectral_signature_defense.py --framework=$framework --durations=20 --durations-min=0 --setup-show
2626
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/detector/poison/test_spectral_signature_defense.py tests"; fi
2727

28-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/preprocessor --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
28+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/preprocessor --framework=$framework --durations=20 --durations-min=0 --setup-show
2929
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/preprocessor tests"; fi
3030

31-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/trainer --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
31+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/trainer --framework=$framework --durations=20 --durations-min=0 --setup-show
3232
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/trainer tests"; fi
3333

34-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/transformer --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
34+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/defences/transformer --framework=$framework --durations=20 --durations-min=0 --setup-show
3535
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/transformer tests"; fi
3636

37-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/preprocessing/audio --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
37+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/preprocessing/audio --framework=$framework --durations=20 --durations-min=0 --setup-show
3838
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/audio tests"; fi
3939

40-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/preprocessing/image --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
40+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/preprocessing/image --framework=$framework --durations=20 --durations-min=0 --setup-show
4141
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/image tests"; fi
4242

43-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/preprocessing/expectation_over_transformation --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
43+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/preprocessing/expectation_over_transformation --framework=$framework --durations=20 --durations-min=0 --setup-show
4444
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/expectation_over_transformation tests"; fi
4545

46-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/utils --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
46+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/utils --framework=$framework --durations=20 --durations-min=0 --setup-show
4747
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed utils tests"; fi
4848

49-
pytest --cov-report=xml --cov=art --cov-append -q -vv -s tests/attacks/poison/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
49+
pytest --cov-report=xml --cov=art --cov-append -q -vv -s tests/attacks/poison/ --framework=$framework --durations=20 --durations-min=0 --setup-show
5050
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/poison tests"; fi
5151

52-
pytest --cov-report=xml --cov=art --cov-append -q -vv -s tests/attacks/evasion/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
52+
pytest --cov-report=xml --cov=art --cov-append -q -vv -s tests/attacks/evasion/ --framework=$framework --durations=20 --durations-min=0 --setup-show
5353
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion"; fi
5454

55-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/speech_recognition/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
55+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/speech_recognition/ --framework=$framework --durations=20 --durations-min=0 --setup-show
5656
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/speech_recognition tests"; fi
5757

58-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/attacks/inference/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
58+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/attacks/inference/ --framework=$framework --durations=20 --durations-min=0 --setup-show
5959
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/inference"; fi
6060

61-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/classifiersFrameworks/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
61+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/classifiersFrameworks/ --framework=$framework --durations=20 --durations-min=0 --setup-show
6262
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed classifiersFrameworks tests"; fi
6363

64-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/classification/test_deeplearning_common.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
64+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/classification/test_deeplearning_common.py --framework=$framework --durations=20 --durations-min=0 --setup-show
6565
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification/test_deeplearning_common.py $framework"; fi
6666

67-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/classification/test_deeplearning_specific.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
67+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/classification/test_deeplearning_specific.py --framework=$framework --durations=20 --durations-min=0 --setup-show
6868
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification tests for framework $framework"; fi
6969

70-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/certification/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
70+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/certification/ --framework=$framework --durations=20 --durations-min=0 --setup-show
7171
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/certification tests for framework $framework"; fi
7272

73-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/classification/test_blackbox_existing_predictions.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
73+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/classification/test_blackbox_existing_predictions.py --framework=$framework --durations=20 --durations-min=0 --setup-show
7474
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification/test_blackbox_existing_predictions.py $framework"; fi
7575

76-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/regression/test_blackbox.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
76+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/estimators/regression/test_blackbox.py --framework=$framework --durations=20 --durations-min=0 --setup-show
7777
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/regression/test_blackbox.py $framework"; fi
7878

79-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/metrics/privacy --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
79+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/metrics/privacy --framework=$framework --durations=20 --durations-min=0 --setup-show
8080
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed metrics/privacy tests"; fi
8181

82-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/evaluations --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
82+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/evaluations --framework=$framework --durations=20 --durations-min=0 --setup-show
8383
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed evaluations tests"; fi
8484

85-
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/test_summary_writer.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
85+
pytest --cov-report=xml --cov=art --cov-append -q -vv tests/test_summary_writer.py --framework=$framework --durations=20 --durations-min=0 --setup-show
8686
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed summary writer tests"; fi
8787

8888
else
@@ -174,7 +174,7 @@ else
174174
echo "######################################################################"
175175
echo ${test}
176176
echo "######################################################################"
177-
pytest --cov-report=xml --cov=art --cov-append -q -vv ${test} --durations=20 --durations-min=0 --setup-show --profile-svg
177+
pytest --cov-report=xml --cov=art --cov-append -q -vv ${test} --durations=20 --durations-min=0 --setup-show
178178
if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed $test"; fi
179179
}
180180

‎setup.cfg

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,6 @@ flake8-ignore =
88
# imported but unused
99
*/__init__.py F401
1010
*/preprocessing.py F401
11-
markers =
12-
skip_framework: marks a test to be skipped for specific framework values. Valid values are ("tensorflow" "keras" "pytorch" "scikitlearn")
13-
only_with_platform: DEPRECATED only used for legacy tests. Use skip_framework instead. marks a test to be performed only for a specific framework value
14-
framework_agnostic: marks a test to be agnostic to frameworks and run only for one default framework
15-
skip_module: Skip the test if a module is not available in the current environment
1611

1712
[mypy]
1813
ignore_missing_imports = True

0 commit comments

Comments
 (0)