Improve testing setup

Signed-off-by: Beat Buesser <[email protected]>

Author: beat-buesser

art/attacks/evasion/laser_attack/laser_attack.py

@@ -24,6 +24,7 @@
 
 from collections.abc import Callable
 import logging
+import math
 from typing import Any
 
 import numpy as np
@@ -221,7 +222,7 @@ def __call__(self, x: int, y: int) -> np.ndarray:
         if distance <= self.width / 2.0:
             return self.rgb
         if self.width / 2.0 <= distance <= 5 * self.width:
-            return np.math.sqrt(self.width) / np.math.pow(distance, 2) * self.rgb  # type: ignore
+            return math.sqrt(self.width) / math.pow(distance, 2) * self.rgb  # type: ignore
 
         return np.array([0.0, 0.0, 0.0])
 

art/attacks/evasion/laser_attack/utils.py

@@ -25,6 +25,7 @@
 from abc import ABC, abstractmethod
 from collections.abc import Callable
 from logging import Logger
+import math
 from pathlib import Path
 import string
 from typing import Any
@@ -46,7 +47,7 @@ def __init__(self, angle: float, bias: float):
         self.bias = bias
 
     def __call__(self, x: float) -> float:
-        return np.math.tan(self.angle) * x + self.bias  # type: ignore
+        return math.tan(self.angle) * x + self.bias  # type: ignore
 
     def distance_of_point_from_the_line(self, x: float, y: float) -> float:
         """
@@ -58,8 +59,8 @@ def distance_of_point_from_the_line(self, x: float, y: float) -> float:
         :returns: Distance.
         """
         y_difference = np.abs(self(x) - y)
-        slope_squared = np.math.pow(np.math.tan(self.angle), 2)  # type: ignore
-        return y_difference / np.math.sqrt(1.0 + slope_squared)  # type: ignore
+        slope_squared = math.pow(math.tan(self.angle), 2)  # type: ignore
+        return y_difference / math.sqrt(1.0 + slope_squared)  # type: ignore
 
     def to_numpy(self) -> np.ndarray:
         """

conftest.py

@@ -495,29 +495,22 @@ def _image_dl_estimator(functional=False, **kwargs):
 @pytest.fixture
 def art_warning(request):
     def _art_warning(exception):
-        if type(exception) is ARTTestFixtureNotImplemented:
-            if request.node.get_closest_marker("framework_agnostic"):
-                if not request.node.get_closest_marker("parametrize"):
-                    raise Exception(
-                        "This test has marker framework_agnostic decorator which means it will only be ran "
-                        "once. However the ART test exception was thrown, hence it is never run fully. "
-                    )
-            elif (
-                request.node.get_closest_marker("only_with_platform")
-                and len(request.node.get_closest_marker("only_with_platform").args) == 1
-            ):
+
+        if request.node.get_closest_marker("framework_agnostic"):
+            if not request.node.get_closest_marker("parametrize"):
                 raise Exception(
-                    "This test has marker only_with_platform decorator which means it will only be ran "
-                    "once. However the ARTTestFixtureNotImplemented exception was thrown, hence it is "
-                    "never run fully. "
+                    "This test has marker framework_agnostic decorator which means it will only be ran "
+                    "once. However the ART test exception was thrown, hence it is never run fully. "
                 )
-
-            # NotImplementedErrors are raised in ART whenever a test model does not exist for a specific
-            # model/framework combination. By catching there here, we can provide a report at the end of each
-            # pytest run list all models requiring to be implemented.
-            warnings.warn(UserWarning(exception))
-        else:
-            raise exception
+        elif (
+            request.node.get_closest_marker("only_with_platform")
+            and len(request.node.get_closest_marker("only_with_platform").args) == 1
+        ):
+            raise Exception(
+                "This test has marker only_with_platform decorator which means it will only be ran "
+                "once. However the ARTTestFixtureNotImplemented exception was thrown, hence it is "
+                "never run fully. "
+            )
 
     return _art_warning
 

requirements_test.txt

@@ -2,8 +2,8 @@
 
 # base
 
-numpy
-scipy
+numpy==1.26.4
+scipy==1.12.0
 matplotlib==3.10.3
 scikit-learn==1.6.1
 six==1.17.0
@@ -50,6 +50,7 @@ jax[cpu]==0.4.30
 pytest~=8.3.2
 pytest-mock~=3.14.0
 pytest-cov~=6.1.1
+pytest-profiling==1.8.1
 pylint==3.2.6
 mypy==1.11.1
 pycodestyle==2.12.1

run_tests.sh

@@ -19,70 +19,70 @@ then
     echo "############### Running tests with framework $framework ###############"
     echo "#######################################################################"
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/detector/evasion --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/detector/evasion --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/detector/evasion tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/detector/poison/test_spectral_signature_defense.py --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/detector/poison/test_spectral_signature_defense.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/detector/poison/test_spectral_signature_defense.py tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/preprocessor --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/preprocessor --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/preprocessor tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/trainer --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/trainer --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/trainer tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/transformer --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/defences/transformer --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed defences/transformer tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/audio --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/audio --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/audio tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/image --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/image --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/image tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/expectation_over_transformation --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/preprocessing/expectation_over_transformation --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed preprocessing/expectation_over_transformation tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/utils --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/utils --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed utils tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/poison/ --framework=$framework  --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/poison/ --framework=$framework  --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/poison tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/evasion/ --framework=$framework  --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv -s tests/attacks/evasion/ --framework=$framework  --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/evasion"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/speech_recognition/ --framework=$framework  --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/speech_recognition/ --framework=$framework  --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/speech_recognition tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/attacks/inference/ --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/attacks/inference/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed attacks/inference"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append -q -vv tests/classifiersFrameworks/  --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append -q -vv tests/classifiersFrameworks/  --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed classifiersFrameworks tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_common.py --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_common.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification/test_deeplearning_common.py $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_specific.py --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_deeplearning_specific.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification tests for framework $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/certification/ --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/certification/ --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/certification tests for framework $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_blackbox_existing_predictions.py --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/classification/test_blackbox_existing_predictions.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/classification/test_blackbox_existing_predictions.py $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/regression/test_blackbox.py --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/estimators/regression/test_blackbox.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed estimators/regression/test_blackbox.py $framework"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/metrics/privacy --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/metrics/privacy --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed metrics/privacy tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/evaluations --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/evaluations --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed evaluations tests"; fi
 
-    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/test_summary_writer.py --framework=$framework --durations=0
+    pytest --cov-report=xml --cov=art --cov-append  -q -vv tests/test_summary_writer.py --framework=$framework --durations=20 --durations-min=0 --setup-show --profile-svg
     if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed summary writer tests"; fi
 
 else
@@ -174,7 +174,7 @@ else
       echo "######################################################################"
       echo ${test}
       echo "######################################################################"
-      pytest --cov-report=xml --cov=art --cov-append  -q -vv ${test} --durations=0
+      pytest --cov-report=xml --cov=art --cov-append  -q -vv ${test} --durations=20 --durations-min=0 --setup-show --profile-svg
       if [[ $? -ne 0 ]]; then exit_code=1; echo "Failed $test"; fi
     }
 

setup.cfg

@@ -1,7 +1,7 @@
 [metadata]
 description-file = README.md
 
-[tool:pytest]
+[tool:flake8]
 flake8-max-line-length = 120
 flake8-ignore =
     *.py E402 W503 E203 E231 E251 E701