Upgrade to nvcomp 5.1.0.21

ci/build_wheel_libcudf.sh

@@ -28,7 +28,7 @@ rapids-pip-retry install \
 # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735)
 export PIP_NO_BUILD_ISOLATION=0
 
-export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=OFF"
+export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON"
 ./ci/build_wheel.sh "${package_name}" "${package_dir}"
 
 # repair wheels and write to the location that artifact-uploading code expects to find them

cmake/rapids_config.cmake

@@ -4,6 +4,8 @@
 # SPDX-License-Identifier: Apache-2.0
 # cmake-format: on
 # =============================================================================
+set(rapids-cmake-repo "bdice/rapids-cmake")
+set(rapids-cmake-branch "nvcomp-5.1")
 file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version)
 if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]])
   set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}")

conda/environments/all_cuda-129_arch-aarch64.yaml

@@ -40,7 +40,7 @@ dependencies:
 - jupyter_client
 - libcurand-dev
 - libkvikio==26.2.*,>=0.0.0a0
-- libnvcomp-dev==5.0.0.6
+- libnvcomp-dev==5.1.0.21
 - libnvjitlink-dev
 - librdkafka>=2.8.0,<2.9.0a0
 - librmm==26.2.*,>=0.0.0a0

conda/environments/all_cuda-129_arch-x86_64.yaml

@@ -41,7 +41,7 @@ dependencies:
 - libcufile-dev
 - libcurand-dev
 - libkvikio==26.2.*,>=0.0.0a0
-- libnvcomp-dev==5.0.0.6
+- libnvcomp-dev==5.1.0.21
 - libnvjitlink-dev
 - librdkafka>=2.8.0,<2.9.0a0
 - librmm==26.2.*,>=0.0.0a0

conda/environments/all_cuda-130_arch-aarch64.yaml

@@ -40,7 +40,7 @@ dependencies:
 - jupyter_client
 - libcurand-dev
 - libkvikio==26.2.*,>=0.0.0a0
-- libnvcomp-dev==5.0.0.6
+- libnvcomp-dev==5.1.0.21
 - libnvjitlink-dev
 - librdkafka>=2.8.0,<2.9.0a0
 - librmm==26.2.*,>=0.0.0a0

conda/environments/all_cuda-130_arch-x86_64.yaml

@@ -41,7 +41,7 @@ dependencies:
 - libcufile-dev
 - libcurand-dev
 - libkvikio==26.2.*,>=0.0.0a0
-- libnvcomp-dev==5.0.0.6
+- libnvcomp-dev==5.1.0.21
 - libnvjitlink-dev
 - librdkafka>=2.8.0,<2.9.0a0
 - librmm==26.2.*,>=0.0.0a0

conda/recipes/libcudf/conda_build_config.yaml

@@ -26,7 +26,7 @@ flatbuffers_version:
   - "=24.3.25"
 
 nvcomp_version:
-  - "=5.0.0.6"
+  - "=5.1.0.21"
 
 zlib_version:
   - ">=1.2.13"

dependencies.yaml

@@ -220,6 +220,7 @@ files:
       - build_cpp
       - depends_on_libkvikio
       - depends_on_librmm
+      - depends_on_libnvcomp
       - depends_on_rapids_logger
   py_run_libcudf:
     output: pyproject
@@ -229,10 +230,7 @@ files:
     includes:
       - depends_on_libkvikio
       - depends_on_librmm
-      # TODO: restore runtime dependency when we no longer vendor nvcomp
-      # (when nvcomp ships C++ wheels)
-      # https://github.com/rapidsai/build-planning/issues/171
-      # - depends_on_libnvcomp
+      - depends_on_libnvcomp
       - depends_on_rapids_logger
   py_build_pylibcudf:
     output: pyproject
@@ -500,7 +498,21 @@ dependencies:
       - output_types: conda
         packages:
           # Align nvcomp version with rapids-cmake
-          - libnvcomp-dev==5.0.0.6
+          - libnvcomp-dev==5.1.0.21
+    specific:
+      - output_types: [requirements, pyproject]
+        matrices:
+          - matrix:
+              cuda: "12.*"
+            packages:
+              - nvidia-libnvcomp-cu12==5.1.0.21
+          - matrix:
+              cuda: "13.*"
+            packages:
+              - nvidia-libnvcomp-cu13==5.1.0.21
+          - matrix:
+            packages:
+              - nvidia-libnvcomp==5.1.0.21
   rapids_build_skbuild:
     common:
       - output_types: [conda, requirements, pyproject]

python/libcudf/CMakeLists.txt

@@ -40,39 +40,32 @@ set(CUDF_EXPORT_NVCOMP OFF)
 include(../../cpp/cmake/thirdparty/get_nvcomp.cmake)
 
 # Install only the specific libnvcomp.so.* library instead of all nvcomp targets
-if(TARGET nvcomp::nvcomp)
-  get_target_property(is_imported nvcomp::nvcomp IMPORTED)
-  if(is_imported)
-    get_target_property(nvcomp_lib_path nvcomp::nvcomp IMPORTED_LOCATION_RELEASE)
-    # Compute the SOVERSION from the library path
-    get_filename_component(nvcomp_lib_dir ${nvcomp_lib_path} DIRECTORY)
-    get_filename_component(nvcomp_lib_name ${nvcomp_lib_path} NAME)
-    string(REGEX REPLACE "^libnvcomp\\.so\\.([0-9]+).*$" "\\1" nvcomp_soversion_major
-                         ${nvcomp_lib_name}
-    )
-    install(
-      FILES ${nvcomp_lib_path}
-      DESTINATION ${SKBUILD_PLATLIB_DIR}/libcudf/lib64/
-      RENAME libnvcomp.so.${nvcomp_soversion_major}
-    )
+if(NOT USE_NVCOMP_RUNTIME_WHEEL)
+  if(TARGET nvcomp::nvcomp)
+    get_target_property(is_imported nvcomp::nvcomp IMPORTED)
+    if(is_imported)
+      get_target_property(nvcomp_lib_path nvcomp::nvcomp IMPORTED_LOCATION_RELEASE)
+      # Compute the SOVERSION from the library path
+      get_filename_component(nvcomp_lib_dir ${nvcomp_lib_path} DIRECTORY)
+      get_filename_component(nvcomp_lib_name ${nvcomp_lib_path} NAME)
+      string(REGEX REPLACE "^libnvcomp\\.so\\.([0-9]+).*$" "\\1" nvcomp_soversion_major
+                           ${nvcomp_lib_name}
+      )
+      install(
+        FILES ${nvcomp_lib_path}
+        DESTINATION ${SKBUILD_PLATLIB_DIR}/libcudf/lib64/
+        RENAME libnvcomp.so.${nvcomp_soversion_major}
+      )
+    else()
+      message(FATAL_ERROR "nvcomp target must be imported")
+    endif()
   else()
-    message(FATAL_ERROR "nvcomp target must be imported")
+    message(FATAL_ERROR "nvcomp target not found")
   endif()
-else()
-  message(FATAL_ERROR "nvcomp target not found")
 endif()
 
 set(CUDA_STATIC_RUNTIME ON)
 
 set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib)
 
 add_subdirectory(../../cpp cudf-cpp)
-
-if(USE_NVCOMP_RUNTIME_WHEEL)
-  set(rpaths "$ORIGIN/../../nvidia/nvcomp")
-  set_property(
-    TARGET cudf
-    PROPERTY INSTALL_RPATH ${rpaths}
-    APPEND
-  )
-endif()

python/libcudf/libcudf/load.py

@@ -40,18 +40,19 @@ def load_library():
         # them.
         import libkvikio
         import librmm
+        import nvidia.libnvcomp
         import rapids_logger
 
         rapids_logger.load_library()
         librmm.load_library()
         libkvikio.load_library()
+        nvidia.libnvcomp.load_library()
     except ModuleNotFoundError:
         # libcudf's runtime dependency on libkvikio may be satisfied by a natively
         # installed library or a conda package, in which case the import will fail and
         # we assume the library is discoverable on system paths.
         pass
 
-    _load_library("libnvcomp.so.5")
     return _load_library("libcudf.so")
 
 

python/libcudf/pyproject.toml

@@ -29,6 +29,7 @@ classifiers = [
 dependencies = [
     "libkvikio==26.2.*,>=0.0.0a0",
     "librmm==26.2.*,>=0.0.0a0",
+    "nvidia-libnvcomp==5.1.0.21",
     "rapids-logger==0.2.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
@@ -72,5 +73,6 @@ requires = [
     "libkvikio==26.2.*,>=0.0.0a0",
     "librmm==26.2.*,>=0.0.0a0",
     "ninja",
+    "nvidia-libnvcomp==5.1.0.21",
     "rapids-logger==0.2.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.