Merge branch 'main' into rust_dlist_get_set

Author: evgeny-leksikov

.ci/jenkins/lib/build-container-matrix.yaml

@@ -57,8 +57,10 @@ steps:
   - name: Prepare
     run: |
       # Setup podman and dependencies
+      rm -f /etc/containers/storage.conf
+      podman system reset -f || true
       ln -sfT $(type -p podman) /usr/bin/docker
-      yum install -y git gettext
+      yum install -y gettext
 
   - name: Build NIXLBench
     enable: ${ENABLE_NIXLBENCH_BUILD}

.ci/jenkins/lib/build-matrix.yaml

@@ -84,8 +84,8 @@ steps:
     parallel: false
     containerSelector: "{ name: 'podman.*' }"
     run: |
+      # change storage driver to improve build performance
+      rm -f /etc/containers/storage.conf ; podman system reset -f || true
       # symlink podman to docker - scripts works with docker commands
       ln -sfT $(type -p podman) /usr/bin/docker
-      # install git for building container image
-      yum install -y git
       contrib/build-container.sh --build-type debug --no-cache

.ci/jenkins/lib/test-matrix.yaml

@@ -71,6 +71,28 @@ steps:
       # check rdma status
       ibv_devinfo
       #ib_write_bw
+      # Ensure proper MPS configuration: MPS requires EXCLUSIVE_PROCESS compute mode
+      # If MPS is running without EXCLUSIVE_PROCESS mode, that's a misconfiguration that causes hangs
+      GPU_COUNT=$(nvidia-smi -L | wc -l)
+      if [ "$GPU_COUNT" -gt 0 ]; then
+        # Check if MPS is running
+        MPS_RUNNING=false
+        if command -v nvidia-cuda-mps-control >/dev/null 2>&1 && pgrep -f nvidia-cuda-mps-server >/dev/null 2>&1; then
+          MPS_RUNNING=true
+        fi
+
+        if [ "$MPS_RUNNING" = true ]; then
+          # MPS is running - ensure EXCLUSIVE_PROCESS mode is set (required for MPS)
+          for i in $(seq 0 $((GPU_COUNT-1))); do
+            nvidia-smi -i $i -c EXCLUSIVE_PROCESS || true
+          done
+        else
+          # MPS is not running - ensure DEFAULT mode
+          for i in $(seq 0 $((GPU_COUNT-1))); do
+            nvidia-smi -i $i -c DEFAULT || true
+          done
+        fi
+      fi
 
 
   - name: Build GPU Test Environment

.gitlab/build.sh

@@ -117,7 +117,7 @@ fi
 # Add DOCA repository and install packages
 ARCH_SUFFIX=$(if [ "${ARCH}" = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi)
 MELLANOX_OS="$(. /etc/lsb-release; echo ${DISTRIB_ID}${DISTRIB_RELEASE} | tr A-Z a-z | tr -d .)"
-wget --tries=3 --waitretry=5 --no-verbose https://www.mellanox.com/downloads/DOCA/DOCA_v3.1.0/host/doca-host_3.1.0-091000-25.07-${MELLANOX_OS}_${ARCH_SUFFIX}.deb -O doca-host.deb
+wget --tries=3 --waitretry=5 --no-verbose https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host_3.2.0-125000-25.10-${MELLANOX_OS}_${ARCH_SUFFIX}.deb -O doca-host.deb
 $SUDO dpkg -i doca-host.deb
 $SUDO apt-get update
 $SUDO apt-get upgrade -y

.gitlab/test_cpp.sh

@@ -41,13 +41,40 @@ export PATH=${INSTALL_DIR}/bin:$PATH
 export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig:$PKG_CONFIG_PATH
 export NIXL_PLUGIN_DIR=${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins
 
+# Set UCX GDA max system latency to allow GDA on SYS topology
+# TODO: Remove this once CI setups have better GPU-NIC locality
+# export UCX_IB_GDA_MAX_SYS_LATENCY=1us
+
 echo "==== Show system info ===="
 env
 nvidia-smi topo -m || true
 ibv_devinfo || true
 uname -a || true
 cat /sys/devices/virtual/dmi/id/product_name || true
 
+echo "==== NVIDIA Peermem check ===="
+if ! lsmod | grep -q nvidia_peermem; then
+    echo "nvidia_peermem module not loaded"
+fi
+
+if [ -f /sys/kernel/mm/memory_peers/nv_mem/version ]; then
+    cat /sys/kernel/mm/memory_peers/nv_mem/version
+else
+    echo "/sys/kernel/mm/memory_peers/nv_mem/version not found "
+fi
+
+if [ -f /sys/module/nvidia_peermem/version ]; then
+    cat /sys/module/nvidia_peermem/version
+else
+    echo "/sys/module/nvidia_peermem/version not found"
+fi
+
+if [ -f /sys/module/nv_peer_mem/version ]; then
+    cat /sys/module/nv_peer_mem/version
+else
+    echo "/sys/module/nv_peer_mem/version not found"
+fi
+
 echo "==== Running ETCD server ===="
 etcd_port=$(get_next_tcp_port)
 etcd_peer_port=$(get_next_tcp_port)
@@ -89,7 +116,8 @@ kill -s INT $telePID
 # fi
 
 # shellcheck disable=SC2154
-gtest-parallel --workers=1 --serialize_test_cases ./bin/gtest -- --min-tcp-port="$min_gtest_port" --max-tcp-port="$max_gtest_port"
+# TODO: enable PrepGpuSignal and ucxDeviceApi tests once the problem in UCX is fixed
+gtest-parallel --workers=1 --serialize_test_cases ./bin/gtest -- --min-tcp-port="$min_gtest_port" --max-tcp-port="$max_gtest_port" --gtest_filter=-*PrepGpuSignal*:*ucxDeviceApi*
 ./bin/test_plugin
 
 # Run NIXL client-server test

benchmark/nixlbench/README.md

@@ -300,8 +300,8 @@ make -j$(nproc) && sudo make install
 **DOCA (Optional):**
 ```bash
 # Add Mellanox repository and install DOCA
-wget https://www.mellanox.com/downloads/DOCA/DOCA_v3.1.0/host/doca-host_3.1.0-091000-25.07-ubuntu2404_amd64.deb
-sudo dpkg -i doca-host_3.1.0-091000-25.07-ubuntu2404_amd64.deb
+wget https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host_3.2.0-125000-25.10-ubuntu2404_amd64.deb -O doca-host.deb
+sudo dpkg -i doca-host.deb
 sudo apt-get update && sudo apt-get install -y doca-sdk-gpunetio libdoca-sdk-gpunetio-dev
 ```
 

benchmark/nixlbench/contrib/Dockerfile

@@ -55,7 +55,7 @@ RUN apt-get update -y && \
 # Add DOCA repository and install packages
 RUN ARCH_SUFFIX=$(if [ "${ARCH}" = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi) && \
     MELLANOX_OS="$(. /etc/lsb-release; echo ${DISTRIB_ID}${DISTRIB_RELEASE} | tr A-Z a-z | tr -d .)" && \
-    wget --tries=3 --waitretry=5 --no-verbose https://www.mellanox.com/downloads/DOCA/DOCA_v3.1.0/host/doca-host_3.1.0-091000-25.07-${MELLANOX_OS}_${ARCH_SUFFIX}.deb -O doca-host.deb && \
+    wget --tries=3 --waitretry=5 --no-verbose https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host_3.2.0-125000-25.10-${MELLANOX_OS}_${ARCH_SUFFIX}.deb -O doca-host.deb && \
     dpkg -i doca-host.deb && \
     apt-get update && \
     apt-get upgrade -y && \

contrib/Dockerfile

@@ -68,7 +68,7 @@ RUN apt-get update -y && \
 # Add DOCA repository and install packages
 RUN ARCH_SUFFIX=$(if [ "${ARCH}" = "aarch64" ]; then echo "arm64"; else echo "amd64"; fi) && \
     MELLANOX_OS="$(. /etc/lsb-release; echo ${DISTRIB_ID}${DISTRIB_RELEASE} | tr A-Z a-z | tr -d .)" && \
-    wget --tries=3 --waitretry=5 --no-verbose https://www.mellanox.com/downloads/DOCA/DOCA_v3.1.0/host/doca-host_3.1.0-091000-25.07-${MELLANOX_OS}_${ARCH_SUFFIX}.deb -O doca-host.deb && \
+    wget --tries=3 --waitretry=5 --no-verbose https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host_3.2.0-125000-25.10-${MELLANOX_OS}_${ARCH_SUFFIX}.deb -O doca-host.deb && \
     dpkg -i doca-host.deb && \
     apt-get update && \
     apt-get upgrade -y && \

contrib/Dockerfile.manylinux

@@ -152,10 +152,10 @@ RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.
     rm rustup-init && \
     chmod -R a+w $RUSTUP_HOME $CARGO_HOME
 
-RUN wget https://www.mellanox.com/downloads/DOCA/DOCA_v3.1.0/host/doca-host-3.1.0-091000_25.07_rhel89.${ARCH}.rpm && \
-    rpm -i doca-host-3.1.0-091000_25.07_rhel89.${ARCH}.rpm && \
+RUN wget --tries=3 --waitretry=5 https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel8.${ARCH}.rpm -O doca-host.rpm && \
+    rpm -i doca-host.rpm && \
     dnf install -y libnl3-devel && \
-    cd /usr/share/doca-host-3.1.0/repo/Packages/ && \
+    cd /usr/share/doca-host-3.2.0/repo/Packages/ && \
     rpm -ivh --nodeps doca-sdk-common-*rpm && \
     rpm -ivh --nodeps doca-sdk-rdma-*rpm && \
     rpm -ivh --nodeps doca-sdk-verbs-*rpm && \

meson.build

@@ -127,25 +127,6 @@ else
     cuda_wheel_dir = 'nixl_cu12'
 endif
 
-# DOCA GPUNETIO
-if cuda_dep.found()
-  nvcc_cmd = find_program('nvcc', required: false)
-  if nvcc_cmd.found()
-    if nvcc_cmd.version().version_compare('>=12.8') and nvcc_cmd.version().version_compare('<13.0')
-      doca_gpunetio_dep = dependency('doca-gpunetio', required : false)
-    else
-      warning('CUDA version = ' + nvcc_cmd.version() + ', GPUNETIO plugin will be disabled')
-      doca_gpunetio_dep = disabler()
-    endif
-  else
-    warning('nvcc not found, GPUNETIO plugin will be disabled')
-    doca_gpunetio_dep = disabler()
-  endif
-else
-  warning('CUDA not found, GPUNETIO plugin will be disabled')
-  doca_gpunetio_dep = disabler()
-endif
-
 # Check for etcd-cpp-api - use multiple methods for discovery
 etcd_dep = dependency('etcd-cpp-api', required : false)
 etcd_inc_path = get_option('etcd_inc_path')
@@ -213,10 +194,11 @@ nvcc_prog = find_program('nvcc', required: false)
 ucx_gpu_device_api_available = false
 if ucx_dep.found() and cuda_dep.found() and nvcc_prog.found()
     cuda = meson.get_compiler('cuda')
+    # TODO: Expose doca_gpunetio_dep through UCX
     have_gpu_side = cuda.compiles('''
             #include <ucp/api/device/ucp_device_impl.h>
             int main() { return 0; }
-        ''', dependencies : ucx_dep, args: nvcc_flags)
+        ''', dependencies : [ucx_dep, doca_gpunetio_dep], args: nvcc_flags)
 
     have_host_side = cpp.compiles('''
             #include <ucp/api/device/ucp_host.h>
@@ -233,6 +215,7 @@ if ucx_dep.found() and cuda_dep.found() and nvcc_prog.found()
     'GPU-side compile'   : have_gpu_side,
     'Host-side compile'  : have_host_side,
     'nvcc available'     : nvcc_prog.found(),
+    'DOCA GPUNETIO found': doca_gpunetio_dep.found(),
   }, section: 'UCX GPU Device API', bool_yn: true)
 endif