Merge pull request #2065 from Idclip/ax_backend_upgrade

AX backend restructure for LLVM opaque ptr support (LLVM 16+)

Author: Idclip

.github/workflows/ax.yml

@@ -66,18 +66,21 @@ jobs:
     strategy:
       matrix:
         # Only test unified builds (core+ax). weekly CI should test standalone
+        # @note  LLVM 17 builds are incompatible with the vdb python plugin on linux
         config:
+          - { image: '2025-clang19', cxx: 'clang++', build: 'Release', cmake: '' }
+          - { image: '2025-clang19', cxx: 'g++',     build: 'Release', cmake: '' }
+          - { image: '2025-clang19', cxx: 'clang++', build: 'Debug',   cmake: '' }
+          - { image: '2025-clang18', cxx: 'clang++', build: 'Release', cmake: '' }
+          - { image: '2024-clang17', cxx: 'clang++', build: 'Release', cmake: '-DOPENVDB_PYTHON_USE_AX=OFF' }
+          - { image: '2024-clang16', cxx: 'clang++', build: 'Release', cmake: '' }
           - { image: '2023-clang15', cxx: 'clang++', build: 'Release', cmake: '' }
           - { image: '2023-clang15', cxx: 'g++',     build: 'Release', cmake: '' }
-          - { image: '2023-clang15', cxx: 'clang++', build: 'Debug',   cmake: '' }
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
       - name: nanobind
-        #if: contains(matrix.config.image, '2023') == false
         run: ./ci/install_nanobind.sh 2.0.0
-      - name: install_gtest
-        run: ./ci/install_gtest.sh 1.15.2
       - name: timestamp
         id: timestamp
         run: echo "timestamp=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
@@ -94,12 +97,11 @@ jobs:
         run: >
           ./ci/build.sh -v
           --build-type=${{ matrix.config.build }}
-          --components='core,python,bin,axcore,axbin,axtest'
+          --components='core,python,axcore,axbin,axtest'
           --cargs=\"
           ${{ matrix.config.cmake }}
           -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
           -DUSE_EXPLICIT_INSTANTIATION=OFF
-          -DOPENVDB_CXX_STRICT=ON
           \"
       - name: test
         run: cd build && ctest -V
@@ -123,8 +125,12 @@ jobs:
       matrix:
         config:
           # Last macos runner before M1 (macos-14)
+          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '20' }
+          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '19' }
+          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '18' }
+          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '17' }
+          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '16' }
           - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '15' }
-          - { runner: 'macos-13', cxx: 'clang++', build: 'Release', llvm: '13' }
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -136,7 +142,7 @@ jobs:
         run: >
           ./ci/build.sh -v
           --build-type=${{ matrix.config.build }}
-          --components="core,python,bin,axcore,axbin,axtest"
+          --components="core,python,axcore,axbin,axtest"
           --cargs=\"
           -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
           -DUSE_EXPLICIT_INSTANTIATION=OFF

.github/workflows/weekly.yml

@@ -295,8 +295,8 @@ jobs:
       matrix:
         config:
           # Unified
-          - { image: '2023-clang15', cxx: 'clang++', build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
-          - { image: '2023-clang15', cxx: 'g++',     build: 'Release', components: 'core,bin,axcore,axbin,axtest', cmake: '' }
+          - { image: '2025-clang19', cxx: 'clang++', build: 'Release', components: 'core,axcore,axbin,axtest', cmake: '' }
+          - { image: '2025-clang19', cxx: 'g++',     build: 'Release', components: 'core,axcore,axbin,axtest', cmake: '' }
       fail-fast: false
     steps:
       - uses: actions/checkout@v3
@@ -378,7 +378,7 @@ jobs:
       github.event.inputs.type == 'all' ||
       github.event.inputs.type == 'ax'
     runs-on: ${{ (github.repository_owner == 'AcademySoftwareFoundation' && 'windows-2022-8c-32g-300h') || 'windows-latest' }}
-    name: windows-vc:${{ matrix.config.vc }}-type:${{ matrix.config.build }}
+    name: ${{ matrix.config.vc }}-${{ matrix.config.crt }}-${{ matrix.config.build }}-llvm${{ matrix.config.llvm }}
     env:
       VCPKG_DEFAULT_TRIPLET: ${{ matrix.config.vc }}
       # Export this with '' avoid bash treating \ as escape
@@ -392,28 +392,36 @@ jobs:
           # @todo  We don't currently run the axtests with shared builds of ax
           # due to symbol issues using LLVM as a static lib (which is the only
           # option on Windows).
-          - { vc: 'x64-windows',        crt: 'MD',  components: 'core,bin,axcore,axbin,python', build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
-          - { vc: 'x64-windows-static', crt: 'MT',  components: 'core,bin,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
-          - { vc: 'x64-windows-static', crt: 'MTd', components: 'core,bin,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          # @note  currently only test static builds with MT as building LLVM as
+          # a shared lib on windows is a bit dicey
+          - { vc: 'x64-windows-static', llvm: '18.1.0', crt: 'MultiThreaded',      components: 'core,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          - { vc: 'x64-windows-static', llvm: '18.1.0', crt: 'MultiThreadedDebug', components: 'core,axcore,axbin,axtest', build: 'Debug',   cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
+          - { vc: 'x64-windows',        llvm: '18.1.0', crt: 'MultiThreadedDLL',   components: 'core,axcore,axbin',        build: 'Release', cmake: '-DOPENVDB_CORE_STATIC=OFF -DOPENVDB_AX_STATIC=OFF' }
+          - { vc: 'x64-windows-static', llvm: '15.0.0', crt: 'MultiThreaded',      components: 'core,axcore,axbin,axtest', build: 'Release', cmake: '-DOPENVDB_CORE_SHARED=OFF -DOPENVDB_AX_SHARED=OFF -DUSE_STATIC_DEPENDENCIES=ON -DBLOSC_USE_EXTERNAL_SOURCES=ON' }
       fail-fast: false
     steps:
     - uses: actions/checkout@v3
     - name: llvm
-      run: ./ci/install_llvm_windows.sh ${{ matrix.config.crt }}
+      run: ./ci/install_llvm_windows.sh ${{ matrix.config.llvm }} ${{ matrix.config.crt }}
     - name: install
       shell: powershell
-      run: .\ci\install_windows.ps1
+      run: |
+        vcpkg update
+        vcpkg install tbb cppunit
     - name: build
       run: >
         ./ci/build.sh -v
         --config=${{ matrix.config.build }}
         --components="${{ matrix.config.components }}"
         --cargs=\'
         -A x64 -G \"Visual Studio 17 2022\"
+        -DOPENVDB_USE_DELAYED_LOADING=OFF
+        -DUSE_BLOSC=OFF \
+        -DUSE_ZLIB=OFF \
         -DVCPKG_TARGET_TRIPLET=${VCPKG_DEFAULT_TRIPLET}
         -DCMAKE_TOOLCHAIN_FILE=\"${VCPKG_INSTALLATION_ROOT}\\scripts\\buildsystems\\vcpkg.cmake\"
         -DMSVC_COMPRESS_PDB=ON
-        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=ON
+        -DOPENVDB_AX_TEST_CMD_DOWNLOADS=OFF
         -DUSE_EXPLICIT_INSTANTIATION=OFF
         -DLLVM_DIR=\"${HOME}\\llvm_install\\lib\\cmake\\llvm\"
         -DCMAKE_INSTALL_PREFIX=\"${VDB_INSTALL_PREFIX}\"

ci/build.sh

@@ -51,7 +51,8 @@ declare -A COMPONENTS
 COMPONENTS['core']='OPENVDB_BUILD_CORE'
 COMPONENTS['python']='OPENVDB_BUILD_PYTHON_MODULE'
 COMPONENTS['test']='OPENVDB_BUILD_UNITTESTS'
-COMPONENTS['bin']='OPENVDB_BUILD_BINARIES'
+# @todo split out vdb tool from this...
+COMPONENTS['bin']='OPENVDB_BUILD_VDB_PRINT,OPENVDB_BUILD_VDB_LOD,OPENVDB_BUILD_VDB_TOOL,OPENVDB_BUILD_VDB_TOOL_UNITTESTS'
 COMPONENTS['view']='OPENVDB_BUILD_VDB_VIEW'
 COMPONENTS['render']='OPENVDB_BUILD_VDB_RENDER'
 COMPONENTS['hou']='OPENVDB_BUILD_HOUDINI_PLUGIN'
@@ -150,7 +151,11 @@ for comp in "${!COMPONENTS[@]}"; do
             setting="ON"; break
         fi
     done
-    CMAKE_EXTRA+=("-D${COMPONENTS[$comp]}=$setting")
+    OPENVDB_TOGGLES="${COMPONENTS[$comp]}"
+    OPENVDB_TOGGLES=(${OPENVDB_TOGGLES//,/ })
+    for toggle in "${!OPENVDB_TOGGLES[@]}"; do
+        CMAKE_EXTRA+=("-D${OPENVDB_TOGGLES[$toggle]}=$setting")
+    done
 done
 
 ################################################
@@ -182,17 +187,14 @@ cd ${BUILD_DIR}
 set -x
 
 # Note:
-# - print and lod binary options are always on and can be toggles with: OPENVDB_BUILD_BINARIES=ON/OFF
+# - OPENVDB_BUILD_BINARIES is always ON, but each binary will be toggled depending on the provided comps
 # - always enabled the python tests with OPENVDB_BUILD_PYTHON_UNITTESTS if the python module is in use,
 #   regardless of the 'test' component being enabled or not (see the OPENVDB_BUILD_PYTHON_UNITTESTS option).
 cmake \
     -DOPENVDB_USE_DEPRECATED_ABI_10=ON \
     -DOPENVDB_USE_DEPRECATED_ABI_11=ON \
-    -DOPENVDB_BUILD_VDB_PRINT=ON \
-    -DOPENVDB_BUILD_VDB_LOD=ON \
-    -DOPENVDB_BUILD_VDB_TOOL=ON \
-    -DOPENVDB_BUILD_VDB_TOOL_UNITTESTS=ON \
     -DOPENVDB_TOOL_USE_NANO=OFF \
+    -DOPENVDB_BUILD_BINARIES=ON \
     -DOPENVDB_BUILD_PYTHON_UNITTESTS=ON \
     -DMSVC_MP_THREAD_COUNT=${PARMS[-j]} \
     "${CMAKE_EXTRA[@]}" \

ci/install_llvm_windows.sh

@@ -4,26 +4,55 @@
 
 set -ex
 
-LLVM_CRT="$1"
-
-git clone -b llvmorg-14.0.0 --depth 1 https://github.com/llvm/llvm-project.git llvm
+LLVM_VER="$1"
+LLVM_CRT="$2"
+
+# Legacy define for older versions of LLVM
+LLVM_LEGACY_CRT_DEFINE=""
+if [[ "${LLVM_CRT}" == "MultiThreaded" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MT"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDebug" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MTd"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDLL" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MD"
+elif [[ "${LLVM_CRT}" == "MultiThreadedDebugDLL" ]]; then
+    LLVM_LEGACY_CRT_DEFINE="MDd"
+fi
+
+git clone -b llvmorg-${LLVM_VER} --depth 1 https://github.com/llvm/llvm-project.git llvm
 cd llvm
 
 mkdir .build
 cd .build
 
-# @note  currently only test static builds with MT
 cmake -B . -S ../llvm -A x64 -G "Visual Studio 17 2022" -Thost=x64 \
     -DCMAKE_INSTALL_PREFIX="${HOME}/llvm_install" \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DLLVM_ENABLE_ASSERTIONS=OFF \
+    -DLLVM_ENABLE_PROJECTS="" \
+    -DLLVM_ENABLE_RUNTIMES="" \
+    -DLLVM_ENABLE_UNWIND_TABLES=OFF \
     -DLLVM_TARGETS_TO_BUILD="host" \
-    -DLLVM_BUILD_TOOLS=OFF \
     -DLLVM_INCLUDE_BENCHMARKS=OFF \
     -DLLVM_INCLUDE_EXAMPLES=OFF \
     -DLLVM_INCLUDE_TESTS=OFF \
+    -DLLVM_INCLUDE_DOCS=OFF \
     -DLLVM_INCLUDE_TOOLS=OFF \
-    -DLLVM_USE_CRT_RELEASE=${LLVM_CRT}
-
-cmake --build . --config Release --target install
+    -DLLVM_INCLUDE_UTILS=OFF \
+    -DLLVM_BUILD_BENCHMARKS=OFF \
+    -DLLVM_BUILD_EXAMPLES=OFF \
+    -DLLVM_BUILD_TESTS=OFF \
+    -DLLVM_BUILD_DOCS=OFF \
+    -DLLVM_BUILD_TOOLS=OFF \
+    -DLLVM_BUILD_UTILS=OFF \
+    -DLLVM_ENABLE_BINDINGS=OFF \
+    -DLLVM_ENABLE_OCAMLDOC=OFF \
+    -DLLVM_PARALLEL_COMPILE_JOBS=4 \
+    -DLLVM_PARALLEL_LINK_JOBS=4 \
+    -DCMAKE_MSVC_RUNTIME_LIBRARY=${LLVM_CRT} \
+    -DLLVM_USE_CRT_RELEASE=${LLVM_LEGACY_CRT_DEFINE}
+
+cmake --build . --parallel 4 --config Release --target install
 
 cd ..
 

cmake/config/OpenVDBVersions.cmake

@@ -48,7 +48,7 @@ if(NOT DISABLE_DEPENDENCY_VERSION_CHECKS)
   set(MINIMUM_OPENEXR_VERSION 3.1)
   set(MINIMUM_ZLIB_VERSION 1.2.7)
   set(MINIMUM_TBB_VERSION 2020.3)
-  set(MINIMUM_LLVM_VERSION 13.0.0)
+  set(MINIMUM_LLVM_VERSION 15.0.0)
   set(MINIMUM_BLOSC_VERSION 1.17.0)
   set(MINIMUM_GLFW_VERSION 3.1)
 
@@ -85,4 +85,4 @@ set(FUTURE_MINIMUM_LOG4CPLUS_VERSION 2.0)
 set(FUTURE_MINIMUM_PYTHON_VERSION 3.11)
 set(FUTURE_MINIMUM_NUMPY_VERSION 1.26.0)
 # set(FUTURE_MINIMUM_HOUDINI_VERSION 20.0)
-set(FUTURE_MINIMUM_LLVM_VERSION 15.0.0)
+set(FUTURE_MINIMUM_LLVM_VERSION 18.0.0)

doc/dependencies.txt

@@ -66,7 +66,7 @@ OpenEXR        | 3.1     | Latest      | EXR serialization support
 TBB            | 2020.3  | 2020.3      | Threading Building Blocks - template library for task parallelism | Y       | Y        | https://www.threadingbuildingblocks.org
 ZLIB           | 1.2.7   | Latest      | Compression library for disk serialization compression            | Y       | Y        | https://www.zlib.net
 Boost          | 1.80    | 1.82        | Components: iostreams                                             | Y       | Y        | https://www.boost.org
-LLVM           | 13.0.0  | 15.0.0*     | Target-independent code generation                                | Y       | Y        | https://llvm.org/
+LLVM           | 15.0.0* | 18.0.0      | Target-independent code generation                                | Y       | Y        | https://llvm.org/
 Bison          | 3.7.0   | 3.7.0       | General-purpose parser generator                                  | Y       | Y        | https://www.gnu.org/software/gcc
 Flex           | 2.6.4   | 2.6.4       | Fast lexical analyzer generator                                   | Y       | Y        | https://github.com/westes/flex
 Python         | 3.10    | 3.11        | The python interpreter and libraries                              | Y       | Y        | https://www.python.org
@@ -87,10 +87,9 @@ Doxygen        | 1.8.8   | 1.8.11      | Documentation generation from C++
 
 @subsection depKnownIssues Known Issues
 
- - @b LLVM >= 15. LLVM has recently changed their IR pointer data types to be
- completely opaque. Some support is provided in LLVM 15 but tests with AX show
- that this is extremely unstable. AX currently only supports LLVM versions up
- until 14.X.X. See https://llvm.org/docs/OpaquePointers.html for more details.
+ - @b On Linux, LLVM == 17 has symbol issues which makes it incompatible when
+ used with the OpenVDB Python plugin. LLVM versions greater than 18 should be
+ preferred.
 
  - @b GLFW: Crash with GLFW 3.2.1 when used with EGL on Linux. OpenVDB View may
  segfault on closure. Fixed in GLFW 3.3

openvdb/openvdb/python/CMakeLists.txt

@@ -15,6 +15,8 @@ include(GNUInstallDirs)
 ###### OpenVDB Python Options
 
 option(USE_NUMPY "Build the python library with numpy support." OFF)
+option(OPENVDB_PYTHON_USE_AX [=[
+Build the python library with AX support. Defaults to ON when OpenVDB AX is enabled."]=] ${USE_AX})
 option(OPENVDB_PYTHON_WRAP_ALL_GRID_TYPES [=[
 Expose (almost) all of the grid types in the python module. Otherwise, only FloatGrid, BoolGrid and
 Vec3SGrid will be exposed (see, e.g., exportIntGrid() in python/pyIntGrid.cc). Compiling the Python
@@ -39,7 +41,15 @@ else()
   set(OPENVDB_LIB openvdb)
 endif()
 
-if(USE_AX)
+if(OPENVDB_PYTHON_USE_AX)
+  if(CMAKE_SYSTEM_NAME STREQUAL "Linux")
+    get_property(OPENVDB_LLVM_VERSION_MAJOR GLOBAL PROPERTY "OPENVDB_LLVM_VERSION_MAJOR")
+    if(OPENVDB_LLVM_VERSION_MAJOR VERSION_EQUAL 17)
+      message(FATAL_ERROR "LLVM version 17.X, used with OpenVDB AX, is incompatible "
+        "with the OpenVDB python plugin. Use any version of LLVM 18+")
+    endif()
+  endif()
+
   # Link the python module against openvdb_ax
   if(NOT OPENVDB_BUILD_AX)
     find_package(OpenVDB REQUIRED openvdb_ax)
@@ -69,7 +79,7 @@ endif()
 if(USE_NUMPY)
   target_compile_definitions(openvdb_python PUBLIC "-DPY_OPENVDB_USE_NUMPY")
 endif()
-if(USE_AX)
+if(OPENVDB_PYTHON_USE_AX)
   target_link_libraries(openvdb_python PUBLIC ${OPENVDB_AX_LIB})
   target_compile_definitions(openvdb_python PUBLIC "-DPY_OPENVDB_USE_AX")
 endif()
@@ -95,7 +105,7 @@ if(OPENVDB_BUILD_PYTHON_UNITTESTS)
     WORKING_DIRECTORY "${OPENVDB_PYTHON_WORKING_DIR}")
 
   set(OPENVDB_PYTHON_TEST_ENV "")
-  if(USE_AX)
+  if(OPENVDB_PYTHON_USE_AX)
     list(APPEND OPENVDB_PYTHON_TEST_ENV "OPENVDB_TEST_PYTHON_AX=1")
   endif()