From a201fe64491042564f8a3f7d1710f613edf8317b Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 2 Dec 2025 13:51:23 -0600 Subject: [PATCH 1/7] Upgrade to nvCOMP 5.1.0.21 --- conda/environments/all_cuda-129_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-129_arch-x86_64.yaml | 2 +- conda/environments/all_cuda-130_arch-aarch64.yaml | 2 +- conda/environments/all_cuda-130_arch-x86_64.yaml | 2 +- conda/recipes/libcudf/conda_build_config.yaml | 2 +- dependencies.yaml | 7 ++----- 6 files changed, 7 insertions(+), 10 deletions(-) diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 4b0fe62b7fa..e2c7b078334 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -40,7 +40,7 @@ dependencies: - jupyter_client - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 9a405474cd0..57a1cddd528 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-130_arch-aarch64.yaml b/conda/environments/all_cuda-130_arch-aarch64.yaml index f16a10af772..5be12b47136 100644 --- a/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -40,7 +40,7 @@ dependencies: - jupyter_client - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-130_arch-x86_64.yaml b/conda/environments/all_cuda-130_arch-x86_64.yaml index 84cf156efe6..b71059b8439 100644 --- a/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 7acdfdd9698..a852dff96c1 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,7 +26,7 @@ flatbuffers_version: - "=24.3.25" nvcomp_version: - - "=5.0.0.6" + - "=5.1.0.21" zlib_version: - ">=1.2.13" diff --git a/dependencies.yaml b/dependencies.yaml index 5c41605c040..29bd96b10a8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -229,10 +229,7 @@ files: includes: - depends_on_libkvikio - depends_on_librmm - # TODO: restore runtime dependency when we no longer vendor nvcomp - # (when nvcomp ships C++ wheels) - # https://github.com/rapidsai/build-planning/issues/171 - # - depends_on_libnvcomp + - depends_on_libnvcomp - depends_on_rapids_logger py_build_pylibcudf: output: pyproject @@ -500,7 +497,7 @@ dependencies: - output_types: conda packages: # Align nvcomp version with rapids-cmake - - libnvcomp-dev==5.0.0.6 + - libnvcomp-dev==5.1.0.21 rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] From da46b2772aa5586b9175ae0a09e478be2dca7d38 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Tue, 2 Dec 2025 13:52:45 -0600 Subject: [PATCH 2/7] Test with rapids-cmake updates --- cmake/rapids_config.cmake | 2 ++ 1 file changed, 2 insertions(+) diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake index 9cda1f26e43..2d9aa1f467c 100644 --- a/cmake/rapids_config.cmake +++ b/cmake/rapids_config.cmake @@ -4,6 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= +set(rapids-cmake-repo "bdice/rapids-cmake") +set(rapids-cmake-branch "nvcomp-5.1") file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version) if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") From 52198c777b6e81d4fd847bfb8720b4269b60dd64 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 2 Dec 2025 16:55:31 -0500 Subject: [PATCH 3/7] Use nvcomp wheel in CMake --- ci/build_wheel_libcudf.sh | 2 +- python/libcudf/CMakeLists.txt | 38 ++++++++++++++++++----------------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index a25dbb86570..e390351309c 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -28,7 +28,7 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 -export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=OFF" +export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" ./ci/build_wheel.sh "${package_name}" "${package_dir}" # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index e137e961890..eb90e2b4dbb 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -40,26 +40,28 @@ set(CUDF_EXPORT_NVCOMP OFF) include(../../cpp/cmake/thirdparty/get_nvcomp.cmake) # Install only the specific libnvcomp.so.* library instead of all nvcomp targets -if(TARGET nvcomp::nvcomp) - get_target_property(is_imported nvcomp::nvcomp IMPORTED) - if(is_imported) - get_target_property(nvcomp_lib_path nvcomp::nvcomp IMPORTED_LOCATION_RELEASE) - # Compute the SOVERSION from the library path - get_filename_component(nvcomp_lib_dir ${nvcomp_lib_path} DIRECTORY) - get_filename_component(nvcomp_lib_name ${nvcomp_lib_path} NAME) - string(REGEX REPLACE "^libnvcomp\\.so\\.([0-9]+).*$" "\\1" nvcomp_soversion_major - ${nvcomp_lib_name} - ) - install( - FILES ${nvcomp_lib_path} - DESTINATION ${SKBUILD_PLATLIB_DIR}/libcudf/lib64/ - RENAME libnvcomp.so.${nvcomp_soversion_major} - ) +if(NOT USE_NVCOMP_RUNTIME_WHEEL) + if(TARGET nvcomp::nvcomp) + get_target_property(is_imported nvcomp::nvcomp IMPORTED) + if(is_imported) + get_target_property(nvcomp_lib_path nvcomp::nvcomp IMPORTED_LOCATION_RELEASE) + # Compute the SOVERSION from the library path + get_filename_component(nvcomp_lib_dir ${nvcomp_lib_path} DIRECTORY) + get_filename_component(nvcomp_lib_name ${nvcomp_lib_path} NAME) + string(REGEX REPLACE "^libnvcomp\\.so\\.([0-9]+).*$" "\\1" nvcomp_soversion_major + ${nvcomp_lib_name} + ) + install( + FILES ${nvcomp_lib_path} + DESTINATION ${SKBUILD_PLATLIB_DIR}/libcudf/lib64/ + RENAME libnvcomp.so.${nvcomp_soversion_major} + ) + else() + message(FATAL_ERROR "nvcomp target must be imported") + endif() else() - message(FATAL_ERROR "nvcomp target must be imported") + message(FATAL_ERROR "nvcomp target not found") endif() -else() - message(FATAL_ERROR "nvcomp target not found") endif() set(CUDA_STATIC_RUNTIME ON) From f682b07bb34c5bfb996574408c9ea6c9982ab996 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 2 Dec 2025 17:03:48 -0500 Subject: [PATCH 4/7] Remove rpath --- python/libcudf/CMakeLists.txt | 9 --------- 1 file changed, 9 deletions(-) diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index eb90e2b4dbb..d8cea5f1b6f 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -69,12 +69,3 @@ set(CUDA_STATIC_RUNTIME ON) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) add_subdirectory(../../cpp cudf-cpp) - -if(USE_NVCOMP_RUNTIME_WHEEL) - set(rpaths "$ORIGIN/../../nvidia/nvcomp") - set_property( - TARGET cudf - PROPERTY INSTALL_RPATH ${rpaths} - APPEND - ) -endif() From 6b816e2e048512b8d6efa80e3f7c0003e7a15559 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 2 Dec 2025 17:19:02 -0500 Subject: [PATCH 5/7] Install nvidia-libnvcomp wheel --- dependencies.yaml | 14 ++++++++++++++ python/libcudf/pyproject.toml | 1 + 2 files changed, 15 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 29bd96b10a8..7fffd8eedd2 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -498,6 +498,20 @@ dependencies: packages: # Align nvcomp version with rapids-cmake - libnvcomp-dev==5.1.0.21 + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + packages: + - nvidia-libnvcomp-cu12==5.1.0.21 + - matrix: + cuda: "13.*" + packages: + - nvidia-libnvcomp-cu13==5.1.0.21 + - matrix: + packages: + - nvidia-libnvcomp==5.1.0.21 rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 7492c51ad75..89cf68f8565 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ dependencies = [ "libkvikio==26.2.*,>=0.0.0a0", "librmm==26.2.*,>=0.0.0a0", + "nvidia-libnvcomp==5.1.0.21", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From 73213c701d6da48c216f378817d5419806de7a88 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 2 Dec 2025 17:31:08 -0500 Subject: [PATCH 6/7] Oops --- dependencies.yaml | 1 + python/libcudf/pyproject.toml | 1 + 2 files changed, 2 insertions(+) diff --git a/dependencies.yaml b/dependencies.yaml index 7fffd8eedd2..a7e9e605d6e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -220,6 +220,7 @@ files: - build_cpp - depends_on_libkvikio - depends_on_librmm + - depends_on_libnvcomp - depends_on_rapids_logger py_run_libcudf: output: pyproject diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 89cf68f8565..0a4b674ae67 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -73,5 +73,6 @@ requires = [ "libkvikio==26.2.*,>=0.0.0a0", "librmm==26.2.*,>=0.0.0a0", "ninja", + "nvidia-libnvcomp==5.1.0.21", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. From e52b7d3849de2e00a20636820eee5fca120622c0 Mon Sep 17 00:00:00 2001 From: Kyle Edwards Date: Tue, 2 Dec 2025 18:33:15 -0500 Subject: [PATCH 7/7] Load library --- python/libcudf/libcudf/load.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 66840a7925f..42cb6f031c0 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -40,18 +40,19 @@ def load_library(): # them. import libkvikio import librmm + import nvidia.libnvcomp import rapids_logger rapids_logger.load_library() librmm.load_library() libkvikio.load_library() + nvidia.libnvcomp.load_library() except ModuleNotFoundError: # libcudf's runtime dependency on libkvikio may be satisfied by a natively # installed library or a conda package, in which case the import will fail and # we assume the library is discoverable on system paths. pass - _load_library("libnvcomp.so.5") return _load_library("libcudf.so")