Merge pull request #961 from Trusted-AI/feature/rename-skip-framework-marker

Feature/rename skip framework marker

Author: beat-buesser

conftest.py

@@ -74,10 +74,10 @@ def get_default_framework():
 
 def pytest_addoption(parser):
     parser.addoption(
-        "--mlFramework",
+        "--framework",
         action="store",
         default=get_default_framework(),
-        help="ART tests allow you to specify which mlFramework to use. The default mlFramework used is `tensorflow`. "
+        help="ART tests allow you to specify which framework to use. The default framework used is `tensorflow`. "
         "Other options available are {0}".format(art_supported_frameworks),
     )
     parser.addoption(
@@ -643,7 +643,7 @@ def create_test_image(create_test_dir):
 
 @pytest.fixture(scope="session")
 def framework(request):
-    ml_framework = request.config.getoption("--mlFramework")
+    ml_framework = request.config.getoption("--framework")
     if ml_framework == "tensorflow":
         import tensorflow as tf
 
@@ -654,12 +654,12 @@ def framework(request):
 
     if ml_framework not in art_supported_frameworks:
         raise Exception(
-            "mlFramework value {0} is unsupported. Please use one of these valid values: {1}".format(
+            "framework value {0} is unsupported. Please use one of these valid values: {1}".format(
                 ml_framework, " ".join(art_supported_frameworks)
             )
         )
-    # if utils_test.is_valid_framework(mlFramework):
-    #     raise Exception("The mlFramework specified was incorrect. Valid options available
+    # if utils_test.is_valid_framework(framework):
+    #     raise Exception("The framework specified was incorrect. Valid options available
     #     are {0}".format(art_supported_frameworks))
     return ml_framework
 
@@ -755,7 +755,7 @@ def get_mnist_dataset(load_mnist_dataset, mnist_shape):
     np.testing.assert_array_almost_equal(y_test_mnist_original, y_test_mnist, decimal=3)
 
 
-# ART test fixture to skip test for specific mlFramework values
+# ART test fixture to skip test for specific framework values
 # eg: @pytest.mark.only_with_platform("tensorflow")
 @pytest.fixture(autouse=True)
 def only_with_platform(request, framework):
@@ -764,13 +764,13 @@ def only_with_platform(request, framework):
             pytest.skip("skipped on this platform: {}".format(framework))
 
 
-# ART test fixture to skip test for specific mlFramework values
-# eg: @pytest.mark.skipMlFramework("tensorflow", "keras", "pytorch", "scikitlearn",
+# ART test fixture to skip test for specific framework values
+# eg: @pytest.mark.skip_framework("tensorflow", "keras", "pytorch", "scikitlearn",
 # "mxnet", "kerastf", "non_dl_frameworks", "dl_frameworks")
 @pytest.fixture(autouse=True)
 def skip_by_framework(request, framework):
-    if request.node.get_closest_marker("skipMlFramework"):
-        framework_to_skip_list = list(request.node.get_closest_marker("skipMlFramework").args)
+    if request.node.get_closest_marker("skip_framework"):
+        framework_to_skip_list = list(request.node.get_closest_marker("skip_framework").args)
         if "dl_frameworks" in framework_to_skip_list:
             framework_to_skip_list.extend(deep_learning_frameworks)
 

run_tests.sh

@@ -8,61 +8,61 @@ export TF_CPP_MIN_LOG_LEVEL="3"
 
 #NOTE: All the tests should be ran within this loop. All other tests are legacy tests that must be
 # made framework independent to be incorporated within this loop
-mlFrameworkList=("tensorflow" "keras" "pytorch" "scikitlearn" "mxnet" "kerastf")
-mlFramework=$1
+frameworkList=("tensorflow" "keras" "pytorch" "scikitlearn" "mxnet" "kerastf")
+framework=$1
 
-if [[ ${mlFramework} != "legacy" ]]
+if [[ ${framework} != "legacy" ]]
 then
     echo "#######################################################################"
-    echo "############## Running tests with framework $mlFramework ##############"
+    echo "############### Running tests with framework $framework ###############"
     echo "#######################################################################"
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/detector/poison/test_spectral_signature_defense.py --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/detector/poison/test_spectral_signature_defense.py --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/detector/poison/test_spectral_signature_defense.py tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/preprocessor --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/preprocessor --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/preprocessor tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/transformer --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/transformer --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/transformer tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/audio --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/audio --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/audio tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/expectation_over_transformation --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/expectation_over_transformation --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/expectation_over_transformation tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/utils --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/utils --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed utils tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/poison/ --mlFramework=$mlFramework  --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/poison/ --framework=$framework  --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/poison tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/evasion/ --mlFramework=$mlFramework  --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/evasion/ --framework=$framework  --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion/test_shadow_attack.py"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/speech_recognition/ --mlFramework=$mlFramework  --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/speech_recognition/ --framework=$framework  --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/speech_recognition tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/attacks/inference/ --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/attacks/inference/ --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/inference"; fi
 
-    pytest -q -s tests/attacks/evasion/test_brendel_and_bethge.py --mlFramework=$mlFramework --durations=0
+    pytest -q -s tests/attacks/evasion/test_brendel_and_bethge.py --framework=$framework --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion/test_brendel_and_bethge.py"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append -q -vv tests/classifiersFrameworks/  --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append -q -vv tests/classifiersFrameworks/  --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed classifiersFrameworks tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/preprocessor/test_spatial_smoothing_pytorch.py  --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/preprocessor/test_spatial_smoothing_pytorch.py  --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/preprocessor/test_spatial_smoothing_pytorch.py tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_common.py --mlFramework=$mlFramework --skip_travis=True --durations=0
-    if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification/test_deeplearning_common.py $mlFramework"; fi
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_common.py --framework=$framework --skip_travis=True --durations=0
+    if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification/test_deeplearning_common.py $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_specific.py --mlFramework=$mlFramework --skip_travis=True --durations=0
-    if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification tests for framework $mlFramework"; fi
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_specific.py --framework=$framework --skip_travis=True --durations=0
+    if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification tests for framework $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/metrics/privacy --mlFramework=$mlFramework --skip_travis=True --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/metrics/privacy --framework=$framework --skip_travis=True --durations=0
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed metrics/privacy tests"; fi
 else
     declare -a attacks=("tests/attacks/test_adversarial_patch.py" \

setup.cfg

@@ -5,8 +5,8 @@ description-file = README.md
 pep8maxlinelength = 120
 pep8ignore = *.py E402 W503 E203 E231 E251 E701
 markers =
-    skipMlFramework: marks a test to be skipped for specific mlFramework values. Valid values are ("tensorflow" "keras" "pytorch" "scikitlearn")
-    only_with_platform: DEPRECATED only used for legacy tests. Use skipMlFramework instead. marks a test to be performed only for a specific mlFramework values
+    skip_framework: marks a test to be skipped for specific framework values. Valid values are ("tensorflow" "keras" "mxnet" "pytorch" "scikitlearn")
+    only_with_platform: DEPRECATED only used for legacy tests. Use skip_framework instead. marks a test to be performed only for a specific framework value
     framework_agnostic: marks a test to be agnostic to frameworks and run only for one default framework
     skip_travis: Skips a test marked with this decorator if the command line argument skip_travis is set to true
     skip_module: Skip the test if a module is not available in the current environment

tests/attacks/evasion/test_adversarial_asr.py

@@ -45,7 +45,7 @@ def test_is_subclass(self, art_warning):
         except ARTTestException as e:
             art_warning(e)
 
-    @pytest.mark.skipMlFramework("tensorflow1", "mxnet", "kerastf", "non_dl_frameworks")
+    @pytest.mark.skip_framework("tensorflow1", "mxnet", "kerastf", "non_dl_frameworks")
     def test_implements_abstract_methods(self, art_warning, asr_dummy_estimator):
         try:
             CarliniWagnerASR(estimator=asr_dummy_estimator())
@@ -61,7 +61,7 @@ def test_classifier_type_check_fail(self, art_warning):
         except ARTTestException as e:
             art_warning(e)
 
-    @pytest.mark.skipMlFramework("tensorflow1", "mxnet", "kerastf", "non_dl_frameworks")
+    @pytest.mark.skip_framework("tensorflow1", "mxnet", "kerastf", "non_dl_frameworks")
     def test_generate_batch(self, art_warning, mocker, asr_dummy_estimator, audio_data):
         try:
             test_input, test_target = audio_data

tests/attacks/evasion/test_auto_attack.py

@@ -41,7 +41,7 @@ def fix_get_mnist_subset(get_mnist_dataset):
     yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.skipMlFramework("tensorflow1", "keras", "pytorch", "non_dl_frameworks", "mxnet", "kerastf")
+@pytest.mark.skip_framework("tensorflow1", "keras", "pytorch", "non_dl_frameworks", "mxnet", "kerastf")
 def test_generate_default(art_warning, fix_get_mnist_subset, image_dl_estimator):
     try:
         classifier, _ = image_dl_estimator(from_logits=True)
@@ -60,7 +60,7 @@ def test_generate_default(art_warning, fix_get_mnist_subset, image_dl_estimator)
         art_warning(e)
 
 
-@pytest.mark.skipMlFramework("tensorflow1", "keras", "pytorch", "non_dl_frameworks", "mxnet", "kerastf")
+@pytest.mark.skip_framework("tensorflow1", "keras", "pytorch", "non_dl_frameworks", "mxnet", "kerastf")
 def test_generate_attacks_and_targeted(art_warning, fix_get_mnist_subset, image_dl_estimator):
     try:
         classifier, _ = image_dl_estimator(from_logits=True)

tests/attacks/evasion/test_auto_projected_gradient_descent.py

@@ -38,7 +38,7 @@ def fix_get_mnist_subset(get_mnist_dataset):
     yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.skipMlFramework("tensorflow1", "keras", "pytorch", "non_dl_frameworks", "mxnet", "kerastf")
+@pytest.mark.skip_framework("tensorflow1", "keras", "pytorch", "non_dl_frameworks", "mxnet", "kerastf")
 def test_generate(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack):
     try:
         classifier = image_dl_estimator_for_attack(AutoProjectedGradientDescent)

tests/attacks/evasion/test_brendel_and_bethge.py

@@ -30,7 +30,7 @@
 logger = logging.getLogger(__name__)
 
 
-@pytest.mark.skipMlFramework("tensorflow1", "keras", "kerastf", "mxnet", "non_dl_frameworks")
+@pytest.mark.skip_framework("tensorflow1", "keras", "kerastf", "mxnet", "non_dl_frameworks")
 @pytest.mark.parametrize("targeted", [True, False])
 @pytest.mark.parametrize("norm", [1, 2, np.inf, "inf"])
 def test_generate(art_warning, get_default_mnist_subset, image_dl_estimator_for_attack, targeted, norm):

tests/attacks/evasion/test_fast_gradient.py

@@ -147,7 +147,7 @@ def test_minimal_perturbations_images(art_warning, fix_get_mnist_subset, image_d
 
 
 @pytest.mark.parametrize("norm", [np.inf, 1, 2])
-@pytest.mark.skipMlFramework("pytorch")  # temporarily skipping for pytorch until find bug fix in bounded test
+@pytest.mark.skip_framework("pytorch")  # temporarily skipping for pytorch until find bug fix in bounded test
 @pytest.mark.framework_agnostic
 def test_norm_images(art_warning, norm, fix_get_mnist_subset, image_dl_estimator_for_attack):
     try:
@@ -183,7 +183,7 @@ def test_norm_images(art_warning, norm, fix_get_mnist_subset, image_dl_estimator
         art_warning(e)
 
 
-@pytest.mark.skipMlFramework("scikitlearn")  # temporarily skipping for scikitlearn until find bug fix in bounded test
+@pytest.mark.skip_framework("scikitlearn")  # temporarily skipping for scikitlearn until find bug fix in bounded test
 @pytest.mark.parametrize("targeted, clipped", [(True, True), (True, False), (False, True), (False, False)])
 @pytest.mark.framework_agnostic
 def test_tabular(art_warning, tabular_dl_estimator, framework, get_iris_dataset, targeted, clipped):

tests/attacks/evasion/test_feature_adversaries.py

@@ -36,7 +36,7 @@ def fix_get_mnist_subset(get_mnist_dataset):
     yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.skipMlFramework("pytorch")
+@pytest.mark.skip_framework("pytorch")
 @pytest.mark.framework_agnostic
 def test_images(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack):
     try:

tests/attacks/evasion/test_frame_saliency.py

@@ -40,7 +40,7 @@ def fix_get_mnist_subset(get_mnist_dataset):
     yield x_train_mnist[:n_train], y_train_mnist[:n_train], x_test_mnist[:n_test], y_test_mnist[:n_test]
 
 
-@pytest.mark.skipMlFramework("pytorch")
+@pytest.mark.skip_framework("pytorch")
 @pytest.mark.framework_agnostic
 def test_one_shot(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack):
     try:
@@ -61,7 +61,7 @@ def test_one_shot(art_warning, fix_get_mnist_subset, image_dl_estimator_for_atta
         art_warning(e)
 
 
-@pytest.mark.skipMlFramework("pytorch")
+@pytest.mark.skip_framework("pytorch")
 @pytest.mark.framework_agnostic
 def test_iterative_saliency(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack):
     try:
@@ -86,7 +86,7 @@ def test_iterative_saliency(art_warning, fix_get_mnist_subset, image_dl_estimato
         art_warning(e)
 
 
-@pytest.mark.skipMlFramework("pytorch")
+@pytest.mark.skip_framework("pytorch")
 @pytest.mark.framework_agnostic
 def test_iterative_saliency_refresh(art_warning, fix_get_mnist_subset, image_dl_estimator_for_attack):
     try: