CI: run with compute sanitizer (rapidsai#498)

Run compute-sanitizer in CI. 
Because of limited CI resources, we only run with the single-communicator. It only takes 4mins to run the tests.

Also fixing a free-bug in a test (found by compute-sanitizer).

Authors:
  - Mads R. B. Kristensen (https://github.com/madsbk)
  - Niranda Perera (https://github.com/nirandaperera)

Approvers:
  - Kyle Edwards (https://github.com/KyleFromNVIDIA)
  - Niranda Perera (https://github.com/nirandaperera)
  - Peter Andreas Entschev (https://github.com/pentschev)

URL: rapidsai#498

Author: madsbk

.github/workflows/pr.yaml

@@ -17,6 +17,7 @@ jobs:
       - conda-cpp-build
       - conda-cpp-linters
       - conda-cpp-tests
+      - conda-cpp-memcheck
       - conda-python-build
       - conda-python-tests
       - docs-build
@@ -121,6 +122,14 @@ jobs:
       build_type: pull-request
       container-options: "--cap-add CAP_SYS_PTRACE --shm-size=8g --ulimit=nofile=1000000:1000000"
       script: ci/test_cpp.sh
+  conda-cpp-memcheck:
+    secrets: inherit
+    needs: conda-cpp-build
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10
+    with:
+      build_type: pull-request
+      script: "ci/test_cpp_memcheck.sh"
+      node_type: "gpu-l4-latest-1"
   conda-python-build:
     needs: conda-cpp-build
     secrets: inherit

.github/workflows/test.yaml

@@ -33,6 +33,14 @@ jobs:
       date: ${{ inputs.date }}
       script: ci/test_cpp.sh
       sha: ${{ inputs.sha }}
+  conda-cpp-memcheck:
+    secrets: inherit
+    needs: checks
+    uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-25.10
+    with:
+      build_type: pull-request
+      script: "ci/test_cpp_memcheck.sh"
+      node_type: "gpu-l4-latest-1"
   conda-python-tests:
     secrets: inherit
     uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-25.10

ci/test_cpp_memcheck.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+# SPDX-FileCopyrightText: Copyright (c) 2024-2025, NVIDIA CORPORATION & AFFILIATES.
+# SPDX-License-Identifier: Apache-2.0
+
+set -xeuo pipefail
+
+. /opt/conda/etc/profile.d/conda.sh
+
+CPP_CHANNEL=$(rapids-download-conda-from-github cpp)
+
+rapids-logger "Generate C++ testing dependencies"
+rapids-dependency-file-generator \
+  --output conda \
+  --file-key test_cpp \
+  --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" \
+  --prepend-channel "${CPP_CHANNEL}" \
+  | tee env.yaml
+
+rapids-mamba-retry env create --yes -f env.yaml -n test
+
+# Temporarily allow unbound variables for conda activation.
+set +u
+conda activate test
+set -u
+
+RAPIDS_TESTS_DIR=${RAPIDS_TESTS_DIR:-"${PWD}/test-results"}/
+mkdir -p "${RAPIDS_TESTS_DIR}"
+
+rapids-print-env
+
+rapids-logger "Check GPU usage"
+nvidia-smi
+
+# Trap ERR so that `EXITCODE=1` is set when a command fails
+EXITCODE=0
+trap "EXITCODE=1" ERR
+set +e
+
+# Support customizing the ctests' install location
+cd "${INSTALL_PREFIX:-${CONDA_PREFIX:-/usr}}/bin/tests/librapidsmpf/"
+
+rapids-logger "Run librapidsmpf gtests with compute-sanitizer (Single Node)"
+compute-sanitizer --tool memcheck --track-stream-ordered-races=all gtests/single_tests --gtest_filter=-CuptiMonitorTest.*
+
+rapids-logger "Test script exiting with value: $EXITCODE"
+exit ${EXITCODE}

cpp/CMakeLists.txt

@@ -287,17 +287,7 @@ endif()
 # ##################################################################################################
 # * add tests -------------------------------------------------------------------------------------
 if(RAPIDSMPF_BUILD_TESTS)
-  # include CTest module -- automatically calls enable_testing()
   include(CTest)
-
-  # ctest cuda memcheck
-  find_program(CUDA_SANITIZER compute-sanitizer)
-  set(MEMORYCHECK_COMMAND ${CUDA_SANITIZER})
-  set(MEMORYCHECK_TYPE CudaSanitizer)
-  set(CUDA_SANITIZER_COMMAND_OPTIONS "--tool memcheck")
-
-  # Always print verbose output when tests fail if run using `make test`.
-  list(APPEND CMAKE_CTEST_ARGUMENTS "--output-on-failure")
   add_subdirectory(tests)
 endif()
 

cpp/tests/test_rmm_resource_adaptor.cpp

@@ -37,8 +37,8 @@ struct throw_at_limit_resource final : public rmm::mr::device_memory_resource {
         return ptr;
     }
 
-    void do_deallocate(void* ptr, std::size_t, rmm::cuda_stream_view) override {
-        RAPIDSMPF_CUDA_TRY(cudaFree(ptr));
+    void do_deallocate(void* ptr, std::size_t, rmm::cuda_stream_view stream) override {
+        RAPIDSMPF_CUDA_TRY(cudaFreeAsync(ptr, stream.value()));
         allocs.erase(ptr);
     }