diff --git a/ci/build_wheel_libcudf.sh b/ci/build_wheel_libcudf.sh index a25dbb86570..e390351309c 100755 --- a/ci/build_wheel_libcudf.sh +++ b/ci/build_wheel_libcudf.sh @@ -28,7 +28,7 @@ rapids-pip-retry install \ # 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) export PIP_NO_BUILD_ISOLATION=0 -export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=OFF" +export SKBUILD_CMAKE_ARGS="-DUSE_NVCOMP_RUNTIME_WHEEL=ON" ./ci/build_wheel.sh "${package_name}" "${package_dir}" # repair wheels and write to the location that artifact-uploading code expects to find them diff --git a/cmake/rapids_config.cmake b/cmake/rapids_config.cmake index 9cda1f26e43..2d9aa1f467c 100644 --- a/cmake/rapids_config.cmake +++ b/cmake/rapids_config.cmake @@ -4,6 +4,8 @@ # SPDX-License-Identifier: Apache-2.0 # cmake-format: on # ============================================================================= +set(rapids-cmake-repo "bdice/rapids-cmake") +set(rapids-cmake-branch "nvcomp-5.1") file(READ "${CMAKE_CURRENT_LIST_DIR}/../VERSION" _rapids_version) if(_rapids_version MATCHES [[^([0-9][0-9])\.([0-9][0-9])\.([0-9][0-9])]]) set(RAPIDS_VERSION_MAJOR "${CMAKE_MATCH_1}") diff --git a/conda/environments/all_cuda-129_arch-aarch64.yaml b/conda/environments/all_cuda-129_arch-aarch64.yaml index 4b0fe62b7fa..e2c7b078334 100644 --- a/conda/environments/all_cuda-129_arch-aarch64.yaml +++ b/conda/environments/all_cuda-129_arch-aarch64.yaml @@ -40,7 +40,7 @@ dependencies: - jupyter_client - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-129_arch-x86_64.yaml b/conda/environments/all_cuda-129_arch-x86_64.yaml index 9a405474cd0..57a1cddd528 100644 --- a/conda/environments/all_cuda-129_arch-x86_64.yaml +++ b/conda/environments/all_cuda-129_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-130_arch-aarch64.yaml b/conda/environments/all_cuda-130_arch-aarch64.yaml index f16a10af772..5be12b47136 100644 --- a/conda/environments/all_cuda-130_arch-aarch64.yaml +++ b/conda/environments/all_cuda-130_arch-aarch64.yaml @@ -40,7 +40,7 @@ dependencies: - jupyter_client - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/environments/all_cuda-130_arch-x86_64.yaml b/conda/environments/all_cuda-130_arch-x86_64.yaml index 84cf156efe6..b71059b8439 100644 --- a/conda/environments/all_cuda-130_arch-x86_64.yaml +++ b/conda/environments/all_cuda-130_arch-x86_64.yaml @@ -41,7 +41,7 @@ dependencies: - libcufile-dev - libcurand-dev - libkvikio==26.2.*,>=0.0.0a0 -- libnvcomp-dev==5.0.0.6 +- libnvcomp-dev==5.1.0.21 - libnvjitlink-dev - librdkafka>=2.8.0,<2.9.0a0 - librmm==26.2.*,>=0.0.0a0 diff --git a/conda/recipes/libcudf/conda_build_config.yaml b/conda/recipes/libcudf/conda_build_config.yaml index 7acdfdd9698..a852dff96c1 100644 --- a/conda/recipes/libcudf/conda_build_config.yaml +++ b/conda/recipes/libcudf/conda_build_config.yaml @@ -26,7 +26,7 @@ flatbuffers_version: - "=24.3.25" nvcomp_version: - - "=5.0.0.6" + - "=5.1.0.21" zlib_version: - ">=1.2.13" diff --git a/dependencies.yaml b/dependencies.yaml index 5c41605c040..a7e9e605d6e 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -220,6 +220,7 @@ files: - build_cpp - depends_on_libkvikio - depends_on_librmm + - depends_on_libnvcomp - depends_on_rapids_logger py_run_libcudf: output: pyproject @@ -229,10 +230,7 @@ files: includes: - depends_on_libkvikio - depends_on_librmm - # TODO: restore runtime dependency when we no longer vendor nvcomp - # (when nvcomp ships C++ wheels) - # https://github.com/rapidsai/build-planning/issues/171 - # - depends_on_libnvcomp + - depends_on_libnvcomp - depends_on_rapids_logger py_build_pylibcudf: output: pyproject @@ -500,7 +498,21 @@ dependencies: - output_types: conda packages: # Align nvcomp version with rapids-cmake - - libnvcomp-dev==5.0.0.6 + - libnvcomp-dev==5.1.0.21 + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + packages: + - nvidia-libnvcomp-cu12==5.1.0.21 + - matrix: + cuda: "13.*" + packages: + - nvidia-libnvcomp-cu13==5.1.0.21 + - matrix: + packages: + - nvidia-libnvcomp==5.1.0.21 rapids_build_skbuild: common: - output_types: [conda, requirements, pyproject] diff --git a/python/libcudf/CMakeLists.txt b/python/libcudf/CMakeLists.txt index e137e961890..d8cea5f1b6f 100644 --- a/python/libcudf/CMakeLists.txt +++ b/python/libcudf/CMakeLists.txt @@ -40,26 +40,28 @@ set(CUDF_EXPORT_NVCOMP OFF) include(../../cpp/cmake/thirdparty/get_nvcomp.cmake) # Install only the specific libnvcomp.so.* library instead of all nvcomp targets -if(TARGET nvcomp::nvcomp) - get_target_property(is_imported nvcomp::nvcomp IMPORTED) - if(is_imported) - get_target_property(nvcomp_lib_path nvcomp::nvcomp IMPORTED_LOCATION_RELEASE) - # Compute the SOVERSION from the library path - get_filename_component(nvcomp_lib_dir ${nvcomp_lib_path} DIRECTORY) - get_filename_component(nvcomp_lib_name ${nvcomp_lib_path} NAME) - string(REGEX REPLACE "^libnvcomp\\.so\\.([0-9]+).*$" "\\1" nvcomp_soversion_major - ${nvcomp_lib_name} - ) - install( - FILES ${nvcomp_lib_path} - DESTINATION ${SKBUILD_PLATLIB_DIR}/libcudf/lib64/ - RENAME libnvcomp.so.${nvcomp_soversion_major} - ) +if(NOT USE_NVCOMP_RUNTIME_WHEEL) + if(TARGET nvcomp::nvcomp) + get_target_property(is_imported nvcomp::nvcomp IMPORTED) + if(is_imported) + get_target_property(nvcomp_lib_path nvcomp::nvcomp IMPORTED_LOCATION_RELEASE) + # Compute the SOVERSION from the library path + get_filename_component(nvcomp_lib_dir ${nvcomp_lib_path} DIRECTORY) + get_filename_component(nvcomp_lib_name ${nvcomp_lib_path} NAME) + string(REGEX REPLACE "^libnvcomp\\.so\\.([0-9]+).*$" "\\1" nvcomp_soversion_major + ${nvcomp_lib_name} + ) + install( + FILES ${nvcomp_lib_path} + DESTINATION ${SKBUILD_PLATLIB_DIR}/libcudf/lib64/ + RENAME libnvcomp.so.${nvcomp_soversion_major} + ) + else() + message(FATAL_ERROR "nvcomp target must be imported") + endif() else() - message(FATAL_ERROR "nvcomp target must be imported") + message(FATAL_ERROR "nvcomp target not found") endif() -else() - message(FATAL_ERROR "nvcomp target not found") endif() set(CUDA_STATIC_RUNTIME ON) @@ -67,12 +69,3 @@ set(CUDA_STATIC_RUNTIME ON) set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) add_subdirectory(../../cpp cudf-cpp) - -if(USE_NVCOMP_RUNTIME_WHEEL) - set(rpaths "$ORIGIN/../../nvidia/nvcomp") - set_property( - TARGET cudf - PROPERTY INSTALL_RPATH ${rpaths} - APPEND - ) -endif() diff --git a/python/libcudf/libcudf/load.py b/python/libcudf/libcudf/load.py index 66840a7925f..42cb6f031c0 100644 --- a/python/libcudf/libcudf/load.py +++ b/python/libcudf/libcudf/load.py @@ -40,18 +40,19 @@ def load_library(): # them. import libkvikio import librmm + import nvidia.libnvcomp import rapids_logger rapids_logger.load_library() librmm.load_library() libkvikio.load_library() + nvidia.libnvcomp.load_library() except ModuleNotFoundError: # libcudf's runtime dependency on libkvikio may be satisfied by a natively # installed library or a conda package, in which case the import will fail and # we assume the library is discoverable on system paths. pass - _load_library("libnvcomp.so.5") return _load_library("libcudf.so") diff --git a/python/libcudf/pyproject.toml b/python/libcudf/pyproject.toml index 7492c51ad75..0a4b674ae67 100644 --- a/python/libcudf/pyproject.toml +++ b/python/libcudf/pyproject.toml @@ -29,6 +29,7 @@ classifiers = [ dependencies = [ "libkvikio==26.2.*,>=0.0.0a0", "librmm==26.2.*,>=0.0.0a0", + "nvidia-libnvcomp==5.1.0.21", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -72,5 +73,6 @@ requires = [ "libkvikio==26.2.*,>=0.0.0a0", "librmm==26.2.*,>=0.0.0a0", "ninja", + "nvidia-libnvcomp==5.1.0.21", "rapids-logger==0.2.*,>=0.0.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.