ci: require all dependencies, with explicit exceptions (#4898)

lgritz · lgritz · commit e382339a50fc · 2025-09-27T11:11:36.000-07:00
We recently had a problem where our use of a dependency was broken, and
it had gone unnoticed because none of our CI jobs actually had the
dependency present and tested it. This can easily happen because so many
of our dependencies are "optional" -- our build will just give a warning
and silently disable the functionality that would have been supported by
the missing dependency.

We have long had build-time options `OpenImageIO_REQUIRED_DEPS`, to
consider specific dependencies (or "all") required even if they would
ordinarily be optional, and `OpenImageIO_OPTIONAL_DEPS`, to make
exceptions. But we didn't use these in CI.

So this PR makes sets `OpenImageIO_REQUIRED_DEPS=all` to make all
dependencies ostensibly required, and then list all exceptions
explicitly. This should make it much more difficult in the future to
make a mistake where use of a dependency is completely untested in our
CI without our being aware of it. And it gives us a visible "hit list"
of untested or under-tested dependencies to slowly whittle down.

Some changes that came long for the ride:

* checked_find_package: explicitly disabled packages are treated as
optional.

* cuda_macros.cmake: Only look for CUDA on platforms that might
conceivably have it (i.e., don't even look on Mac).

* Separate linux-aswf from linux-ubuntu into separate job groups,
because that makes it easier to have shared commonalities of which
dependencies they test.

Signed-off-by: Larry Gritz &lt;lg@larrygritz.com&gt;
diff --git a/.github/workflows/build-steps.yml b/.github/workflows/build-steps.yml
@@ -68,6 +68,10 @@ on:
         type: string
       nametag:
         type: string
+      required_deps:
+        type: string
+      optional_deps:
+        type: string
     secrets:
         PASSED_GITHUB_TOKEN:
           required: false
@@ -102,6 +106,12 @@ jobs:
       ABI_CHECK: ${{inputs.abi_check}}
       ACTIONS_RUNNER_FORCE_ACTIONS_NODE_VERSION: node16
       ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true
+      # For CI, sometimes we want to require all dependencies to be present,
+      # except for a select few listed explicitly. This ensures that we don't
+      # accidentally have a situation in which we think we are building
+      # against and testing an optional dependency, but in fact are not.
+      OpenImageIO_REQUIRED_DEPS: ${{inputs.required_deps}}
+      OpenImageIO_OPTIONAL_DEPS: ${{inputs.optional_deps}}
 
     steps:
       - name: Checkout repo
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -239,9 +239,9 @@ jobs:
 
 
   #
-  # Linux Tests
+  # Linux Tests using ASWF-docker containers
   #
-  linux:
+  linux-aswf:
     if: ${{ ! contains(github.ref, 'windows-only') && ! contains(github.ref, 'macos-only') }}
     name: "${{matrix.desc}}"
     uses: ./.github/workflows/build-steps.yml
@@ -273,6 +273,10 @@ jobs:
       ctest_test_timeout: ${{ matrix.ctest_test_timeout }}
       coverage: ${{ matrix.coverage || 0 }}
       sonar: ${{ matrix.sonar || 0 }}
+      # Override required_deps to be 'all' and explicitly list as optional
+      # only the ones we are intentionally not testing for those jobs.
+      required_deps: ${{ matrix.required_deps || 'all' }}
+      optional_deps: ${{ matrix.optional_deps || 'DCMTK;JXL;Libheif;Nuke;OpenCV;openjph;OpenVDB;Qt5;R3DSDK;'}}${{matrix.optional_deps_append}}
     strategy:
       fail-fast: false
       matrix:
@@ -287,6 +291,7 @@ jobs:
             fmt_ver: 10.1.1
             pybind11_ver: v2.10.0
             setenvs: export PUGIXML_VERSION=v1.13
+            optional_deps_append: 'LibRaw;Ptex;Qt6'
           - desc: VFX2023 icc/C++17 py3.10 exr3.1 ocio2.3 qt5.15
             nametag: linux-vfx2023.icc
             runner: ubuntu-latest
@@ -303,6 +308,7 @@ jobs:
                             DISABLE_libuhdr=1
             # For icc, use fp-model precise to eliminate needless LSB errors
             # that make test results differ from other platforms.
+            optional_deps_append: "LibRaw;Ptex;Qt6"
           - desc: VFX2023 icx/C++17 py3.10 exr3.1 ocio2.3 qt5.15
             nametag: linux-vfx2023.icx
             runner: ubuntu-latest
@@ -319,6 +325,7 @@ jobs:
                             UHDR_CMAKE_CXX_COMPILER=g++
             # Building libuhdr with icx results in test failures
             # so we force using gcc/g++.
+            optional_deps_append: "LibRaw;Ptex;Qt6"
           - desc: VFX2024 gcc11/C++17 py3.11 exr3.2 ocio2.3
             nametag: linux-vfx2024
             runner: ubuntu-latest
@@ -330,6 +337,7 @@ jobs:
             pybind11_ver: v2.12.0
             benchmark: 1
             setenvs: export PUGIXML_VERSION=v1.14
+            optional_deps_append: "LibRaw"
           - desc: VFX2024 clang/C++17 py3.11 exr3.2 ocio2.3
             nametag: linux-vfx2024.clang
             runner: ubuntu-latest
@@ -343,6 +351,7 @@ jobs:
             pybind11_ver: v2.12.0
             benchmark: 1
             setenvs: export PUGIXML_VERSION=v1.14
+            optional_deps_append: "LibRaw"
           - desc: VFX2025 gcc11/C++17 py3.11 exr3.3 ocio2.4
             nametag: linux-vfx2025
             runner: ubuntu-latest
@@ -354,6 +363,7 @@ jobs:
             pybind11_ver: v2.13.6
             benchmark: 1
             setenvs: export PUGIXML_VERSION=v1.15
+            optional_deps_append: "openjph;Qt6"
           - desc: VFX2026 gcc14/C++20 py3.13 exr3.4 ocio2.4
             nametag: linux-vfx2026
             runner: ubuntu-latest
@@ -364,6 +374,7 @@ jobs:
             pybind11_ver: v3.0.0
             benchmark: 1
             # setenvs: export 
+            optional_deps_append: "Qt5;Qt6"
           - desc: Sanitizers
             nametag: sanitizer
             runner: ubuntu-latest
@@ -378,6 +389,7 @@ jobs:
                             OIIO_CMAKE_FLAGS="-DSANITIZE=address,undefined -DOIIO_HARDENING=3 -DUSE_PYTHON=0"
                             CTEST_EXCLUSIONS="broken|png-damaged"
                             OpenImageIO_BUILD_LOCAL_DEPS=PNG
+            optional_deps_append: "LibRaw"
 
           # Test ABI stability. `abi_check` is the version or commit that we
           # believe is the current standard against which we don't want to
@@ -397,7 +409,52 @@ jobs:
             abi_check: 9bfcce725a3806a3f70c7e838d9d98d6d95c917a
             setenvs: export OIIO_CMAKE_FLAGS="-DOIIO_BUILD_TOOLS=0 -DOIIO_BUILD_TESTS=0 -DUSE_PYTHON=0"
                             USE_OPENCV=0 USE_FFMPEG=0 USE_PYTHON=0 USE_FREETYPE=0
+            optional_deps_append: "openjph;Qt6"
 
+
+  #
+  # Linux Tests using GHA Ubuntu runners directly
+  #
+  linux-ubuntu:
+    if: ${{ ! contains(github.ref, 'windows-only') && ! contains(github.ref, 'macos-only') }}
+    name: "${{matrix.desc}}"
+    uses: ./.github/workflows/build-steps.yml
+    with:
+      nametag: ${{ matrix.nametag || 'unnamed!' }}
+      runner: ${{ matrix.runner || 'ubuntu-latest' }}
+      container: ${{ matrix.container }}
+      cc_compiler: ${{ matrix.cc_compiler }}
+      cxx_compiler: ${{ matrix.cxx_compiler }}
+      cxx_std: ${{ matrix.cxx_std || '17' }}
+      build_type: ${{ matrix.build_type || 'Release' }}
+      depcmds: ${{ matrix.depcmds }}
+      extra_artifacts: ${{ matrix.extra_artifacts }}
+      fmt_ver: ${{ matrix.fmt_ver }}
+      opencolorio_ver: ${{ matrix.opencolorio_ver }}
+      openexr_ver: ${{ matrix.openexr_ver }}
+      pybind11_ver: ${{ matrix.pybind11_ver }}
+      python_ver: ${{ matrix.python_ver }}
+      setenvs: ${{ matrix.setenvs }}
+      simd: ${{ matrix.simd }}
+      skip_build: ${{ matrix.skip_build }}
+      skip_tests: ${{ matrix.skip_tests }}
+      abi_check: ${{ matrix.abi_check }}
+      benchmark: ${{ matrix.benchmark }}
+      build_docs: ${{ matrix.build_docs }}
+      clang_format: ${{ matrix.clang_format }}
+      generator: ${{ matrix.generator }}
+      ctest_args: ${{ matrix.ctest_args }}
+      ctest_test_timeout: ${{ matrix.ctest_test_timeout }}
+      coverage: ${{ matrix.coverage || 0 }}
+      sonar: ${{ matrix.sonar || 0 }}
+      # Override required_deps to be 'all' and explicitly list as optional
+      # only the ones we are intentionally not testing for those jobs.
+      required_deps: ${{ matrix.required_deps || 'all' }}
+      optional_deps: ${{ matrix.optional_deps || 'CUDAToolkit;DCMTK;JXL;Nuke;OpenGL;openjph;OpenVDB;Ptex;pystring;Qt5;R3DSDK;' }}${{matrix.optional_deps_append}}
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
           # Test formatting. This test entry doesn't build at all, it
           # just runs clang-format on everything, and passes if nothing is
           # misformatted. Upon failure, the build artifact will be the full
@@ -440,6 +497,10 @@ jobs:
                             WEBP_VERSION=v1.6.0
                             FREETYPE_VERSION=VER-2-14-0
                             USE_OPENVDB=0
+            # Ensure we are testing all the deps we think we are. We would
+            # like this test to have minimal missing dependencies.
+            required_deps: all
+            optional_deps: 'CUDAToolkit;DCMTK;JXL;Nuke;OpenCV;OpenGL;OpenVDB;R3DSDK'
           - desc: bleeding edge gcc14 C++23 py3.12 OCIO/libtiff/exr-main avx2
             nametag: linux-bleeding-edge
             runner: ubuntu-24.04
@@ -467,6 +528,10 @@ jobs:
                             FREETYPE_VERSION=master
                             QT_VERSION=0 INSTALL_OPENCV=0
                             # The installed OpenVDB has a TLS conflict with Python 3.8
+            # Ensure we are testing all the deps we think we are. We would
+            # like this test to have minimal missing dependencies.
+            required_deps: all
+            optional_deps: 'CUDAToolkit;DCMTK;JXL;Nuke;OpenCV;OpenGL;OpenVDB;R3DSDK'
           - desc: all local builds gcc12 C++17 avx2 exr3.2 ocio2.3
             nametag: linux-local-builds
             runner: ubuntu-22.04
@@ -482,7 +547,6 @@ jobs:
                             PTEX_VERSION=v2.4.2
                             PUGIXML_VERSION=v1.14
                             WEBP_VERSION=v1.4.0
-
           - desc: clang15 C++17 avx2 exr3.1 ocio2.3
             nametag: linux-clang15
             runner: ubuntu-22.04
@@ -597,6 +661,11 @@ jobs:
       ctest_test_timeout: ${{ matrix.ctest_test_timeout || '800' }}
       coverage: ${{ matrix.coverage || 0 }}
       sonar: ${{ matrix.sonar || 0 }}
+      # We're able to use Homebrew to install ALMOST every dependency, so the
+      # only optional ones in our Mac CI tests are commercial things we can't
+      # test in GHA CI.
+      required_deps: ${{ matrix.required_deps || 'all' }}
+      optional_deps: ${{ matrix.optional_deps || 'Nuke;R3DSDK;' }}${{matrix.optional_deps_append}}
     strategy:
       fail-fast: false
       matrix:
@@ -663,6 +732,11 @@ jobs:
       ctest_test_timeout: ${{ matrix.ctest_test_timeout }}
       coverage: ${{ matrix.coverage || 0 }}
       sonar: ${{ matrix.sonar || 0 }}
+      # Windows is a PITA, so we expect very few dependencies to be present or
+      # built. But we would like to add more dependencies and reduce this list
+      # of exceptions in the future.
+      required_deps: ${{ matrix.required_deps || 'all' }}
+      optional_deps: ${{ matrix.optional_deps || 'CUDAToolkit;DCMTK;FFmpeg;GIF;JXL;Libheif;LibRaw;Nuke;OpenCV;OpenGL;OpenJPEG;openjph;OpenCV;OpenVDB;Ptex;pystring;Qt5;Qt6;TBB;R3DSDK;${{matrix.optional_deps_append}}' }}
     strategy:
       fail-fast: false
       matrix:
diff --git a/src/build-scripts/gh-installdeps.bash b/src/build-scripts/gh-installdeps.bash
@@ -17,22 +17,28 @@ if [[ "$ASWF_ORG" != ""  ]] ; then
 
     #ls /etc/yum.repos.d
 
-    if [[ "$ASWF_VFXPLATFORM_VERSION" == "2021" || "$ASWF_VFXPLATFORM_VERSION" == "2022" ]] ; then
-        # CentOS 7 based containers need the now-nonexistant centos repo to be
+    # time sudo dnf upgrade --refresh || true
+    time sudo dnf install --nogpgcheck https://mirrors.rpmfusion.org/free/el/rpmfusion-free-release-$(rpm -E %rhel).noarch.rpm -y || true
+
+    if [[ "$ASWF_VFXPLATFORM_VERSION" == "2022" ]] ; then
+        # CentOS 7 based containers need the now-nonexistent centos repo to be
         # excluded or all the subsequent yum install commands will fail.
         yum-config-manager --disable centos-sclo-rh || true
         sed -i 's,^mirrorlist=,#,; s,^#baseurl=http://mirror\.centos\.org/centos/$releasever,baseurl=https://vault.centos.org/7.9.2009,' /etc/yum.repos.d/CentOS-Base.repo
     fi
 
-    sudo yum install -y giflib giflib-devel || true
+    time time sudo yum install -y giflib giflib-devel || true
     if [[ "${USE_OPENCV}" != "0" ]] ; then
-        sudo yum install -y opencv opencv-devel || true
+        time sudo yum install -y opencv opencv-devel || true
     fi
     if [[ "${USE_FFMPEG}" != "0" ]] ; then
-        sudo yum install -y ffmpeg ffmpeg-devel || true
+        time sudo dnf install -y ffmpeg ffmpeg-devel || true
     fi
     if [[ "${USE_FREETYPE:-1}" != "0" ]] ; then
-        sudo yum install -y freetype freetype-devel || true
+        time sudo yum install -y freetype freetype-devel || true
+    fi
+    if [[ "${USE_LIBRAW:-0}" != "0" ]] ; then
+        time sudo yum install -y LibRaw LibRaw-devel || true
     fi
     if [[ "${EXTRA_DEP_PACKAGES}" != "" ]] ; then
         time sudo yum install -y ${EXTRA_DEP_PACKAGES} || true
diff --git a/src/cmake/cuda_macros.cmake b/src/cmake/cuda_macros.cmake
@@ -3,11 +3,11 @@
 # https://github.com/AcademySoftwareFoundation/OpenImageIO
 
 
-set_option (OIIO_USE_CUDA "Include Cuda support if found" OFF)
+set_option (OIIO_USE_CUDA "Include CUDA support if found" OFF)
 set_cache (CUDA_TARGET_ARCH "sm_60" "CUDA GPU architecture (e.g. sm_60)")
 set_cache (CUDAToolkit_ROOT "" "Path to CUDA toolkit")
 
-if (OIIO_USE_CUDA)
+if (OIIO_USE_CUDA AND NOT APPLE)
     set (CUDA_PROPAGATE_HOST_FLAGS ON)
     set (CUDA_VERBOSE_BUILD ${VERBOSE})
     checked_find_package(CUDAToolkit
diff --git a/src/cmake/dependency_utils.cmake b/src/cmake/dependency_utils.cmake
@@ -76,7 +76,7 @@ function (dump_matching_variables pattern)
 endfunction ()
 
 
-# Helper: Print a report about missing dependencies and give insructions on
+# Helper: Print a report about missing dependencies and give instructions on
 # how to turn on automatic local dependency building.
 function (print_package_notfound_report)
     message (STATUS)
@@ -286,7 +286,7 @@ macro (checked_find_package pkgname)
     #
     cmake_parse_arguments(_pkg   # prefix
         # noValueKeywords:
-        "REQUIRED;CONFIG;PREFER_CONFIG;DEBUG;NO_RECORD_NOTFOUND;NO_FP_RANGE_CHECK"
+        "REQUIRED;OPTIONAL;CONFIG;PREFER_CONFIG;DEBUG;NO_RECORD_NOTFOUND;NO_FP_RANGE_CHECK"
         # singleValueKeywords:
         "ENABLE;ISDEPOF;VERSION_MIN;VERSION_MAX;RECOMMEND_MIN;RECOMMEND_MIN_REASON;BUILD_LOCAL"
         # multiValueKeywords:
@@ -303,17 +303,24 @@ macro (checked_find_package pkgname)
         set (${pkgname}_FIND_QUIETLY true)
         set (${pkgname_upper}_FIND_QUIETLY true)
     endif ()
-    if ("${pkgname}" IN_LIST ${PROJECT_NAME}_REQUIRED_DEPS OR "ALL" IN_LIST ${PROJECT_NAME}_REQUIRED_DEPS)
+    if ("${pkgname}" IN_LIST ${PROJECT_NAME}_REQUIRED_DEPS
+        OR "ALL" IN_LIST ${PROJECT_NAME}_REQUIRED_DEPS
+        OR "all" IN_LIST ${PROJECT_NAME}_REQUIRED_DEPS)
         set (_pkg_REQUIRED 1)
     endif ()
-    if ("${pkgname}" IN_LIST ${PROJECT_NAME}_OPTIONAL_DEPS OR "ALL" IN_LIST ${PROJECT_NAME}_OPTIONAL_DEPS)
+    if ("${pkgname}" IN_LIST ${PROJECT_NAME}_OPTIONAL_DEPS
+        OR "ALL" IN_LIST ${PROJECT_NAME}_OPTIONAL_DEPS
+        OR "all" IN_LIST ${PROJECT_NAME}_OPTIONAL_DEPS)
         set (_pkg_REQUIRED 0)
+        set (_pkg_OPTIONAL 1)
     endif ()
     # string (TOLOWER "${_pkg_BUILD_LOCAL}" _pkg_BUILD_LOCAL)
     if ("${pkgname}" IN_LIST ${PROJECT_NAME}_BUILD_LOCAL_DEPS
+        OR ${PROJECT_NAME}_BUILD_LOCAL_DEPS STREQUAL "ALL"
         OR ${PROJECT_NAME}_BUILD_LOCAL_DEPS STREQUAL "all")
         set (_pkg_BUILD_LOCAL "always")
     elseif ("${pkgname}" IN_LIST ${PROJECT_NAME}_BUILD_MISSING_DEPS
+            OR ${PROJECT_NAME}_BUILD_MISSING_DEPS STREQUAL "ALL"
             OR ${PROJECT_NAME}_BUILD_MISSING_DEPS STREQUAL "all")
         set_if_not (_pkg_BUILD_LOCAL "missing")
     endif ()
@@ -341,6 +348,11 @@ macro (checked_find_package pkgname)
             set (_quietskip true)
         endif ()
     endif ()
+    if (NOT _enable)
+        set (_pkg_OPTIONAL 1)
+        set (_pkg_REQUIRED 0)
+        message(STATUS "Forcing optional of disabled ${pkgname}")
+    endif ()
     set (_config_status "")
     unset (_${pkgname}_version_range)
     if (_pkg_BUILD_LOCAL AND NOT _pkg_NO_FP_RANGE_CHECK)
@@ -361,7 +373,7 @@ macro (checked_find_package pkgname)
     #
     set (${pkgname}_FOUND FALSE)
     set (${pkgname}_LOCAL_BUILD FALSE)
-    if (_enable OR _pkg_REQUIRED)
+    if (_enable OR (_pkg_REQUIRED AND NOT _pkg_OPTIONAL))
         # Unless instructed not to, try to find the package externally
         # installed.
         if (${pkgname}_FOUND OR ${pkgname_upper}_FOUND OR _pkg_BUILD_LOCAL STREQUAL "always")
