testing: env variable OSL_TESTSUITE_THRESH_SCALE scales test thresholds (#1934)

For any individual test running, the OSL_TESTSUITE_THRESH_SCALE
environment variable, if set, will scale the image comparison
thresholds used to decide if a test passes.

Furthermore, a special OSL_TESTSUITE_OPTIX_THRESH_SCALE environment
variable, if set, will cause *all* OptiX related tests to be set up
with scaled thresholds.

Signed-off-by: Larry Gritz <[email protected]>

Author: lgritz

src/cmake/testing.cmake

@@ -90,6 +90,9 @@ macro (add_one_testsuite testname testsrcdir)
             # Make sure libnvrtc-builtins.so is reachable
             set_property (TEST ${testname} APPEND PROPERTY ENVIRONMENT LD_LIBRARY_PATH=${CUDA_TOOLKIT_ROOT_DIR}/lib64:$ENV{LD_LIBRARY_PATH})
         endif()
+        if (DEFINED ENV{OSL_TESTSUITE_OPTIX_THRESH_SCALE})
+            set_property (TEST ${testname} APPEND PROPERTY ENVIRONMENT OSL_TESTSUITE_THRESH_SCALE=$ENV{OSL_TESTSUITE_OPTIX_THRESH_SCALE})
+        endif ()
     endif ()
     if (${testname} MATCHES "-reg")
         # These are batched shading regression tests. Some are quite

testsuite/runtest.py

@@ -406,6 +406,16 @@ def runtest (command, outputs, failureok=0, failthresh=0, failpercent=0, regress
     hardfail *= 2.0
     failpercent *= 2.0
 
+# Allow an environment variable to scale the testsuite image comparison
+# thresholds:
+if 'OSL_TESTSUITE_THRESH_SCALE' in os.environ :
+    thresh_scale = float(os.getenv('OSL_TESTSUITE_THRESH_SCALE', '1.0'))
+    failthresh *= thresh_scale
+    hardfail *= thresh_scale
+    failpercent *= thresh_scale
+    failrelative *= thresh_scale
+    allowfailures = int(allowfailures * thresh_scale)
+
 # Force out.txt to be in the outputs
 ##if "out.txt" not in outputs :
 ##    outputs.append ("out.txt")