Merge pull request #59 from projectchrono/copilot/update-demsovler-gpu-selection

Add GPU selection constructor overloads and migrate ownership to unique_ptr

Author: Ruochun

_codeql_detected_source_root

@@ -0,0 +1 @@
+.

src/DEM/API.h

@@ -10,6 +10,7 @@
 #include <set>
 #include <cfloat>
 #include <functional>
+#include <memory>
 
 #include "kT.h"
 #include "dT.h"
@@ -48,7 +49,11 @@ class DEMTracker;
 /// Main DEM-Engine solver.
 class DEMSolver {
   public:
+    /// Default constructor: scans available GPUs and uses at most 2.
     DEMSolver(unsigned int nGPUs = 2);
+    /// Construct using explicit GPU device IDs. Errors if any ID is unavailable. Warns and uses the first 2 if more
+    /// than 2 are given.
+    DEMSolver(std::vector<int> device_ids);
     ~DEMSolver();
 
     /// Set output detail level.
@@ -1806,12 +1811,12 @@ class DEMSolver {
     // DEM system's workers, helpers, friends
     ////////////////////////////////////////////////////////////////////////////////
 
-    WorkerReportChannel* kTMain_InteractionManager;
-    WorkerReportChannel* dTMain_InteractionManager;
-    GpuManager* dTkT_GpuManager;
-    ThreadManager* dTkT_InteractionManager;
-    DEMKinematicThread* kT;
-    DEMDynamicThread* dT;
+    std::unique_ptr<GpuManager> dTkT_GpuManager;
+    std::unique_ptr<ThreadManager> dTkT_InteractionManager;
+    std::unique_ptr<WorkerReportChannel> dTMain_InteractionManager;
+    std::unique_ptr<WorkerReportChannel> kTMain_InteractionManager;
+    std::unique_ptr<DEMDynamicThread> dT;
+    std::unique_ptr<DEMKinematicThread> kT;
 
     ////////////////////////////////////////////////////////////////////////////////
     // DEM system's private methods

src/DEM/APIPrivate.cpp

@@ -579,7 +579,9 @@ void DEMSolver::decideCDMarginStrat() {
 
 void DEMSolver::reportInitStats() const {
     DEME_INFO("\n");
-    DEME_INFO("Number of total active devices: %d", dTkT_GpuManager->getNumDevices());
+
+    DEME_INFO("Number of system devices detected: %d", GpuManager::scanNumDevices());
+    DEME_INFO("Number of active devices used by DEME: %d", dTkT_GpuManager->getNumDevices());
 
     DEME_INFO("User-specified X-dimension range: [%.7g, %.7g]", m_user_box_min.x, m_user_box_max.x);
     DEME_INFO("User-specified Y-dimension range: [%.7g, %.7g]", m_user_box_min.y, m_user_box_max.y);
@@ -1280,8 +1282,8 @@ void DEMSolver::packDataPointers() {
     kT->packDataPointers();
     // Each worker thread needs pointers used for data transfering. Note this step must be done after packDataPointers
     // are called, so each thread has its own pointers packed.
-    dT->packTransferPointers(kT);
-    kT->packTransferPointers(dT);
+    dT->packTransferPointers(kT.get());
+    kT->packTransferPointers(dT.get());
     // Finally, the API needs to map all mesh to their owners
     for (const auto& mmesh : m_meshes) {
         m_owner_mesh_map[mmesh->owner] = mmesh->cache_offset;
@@ -1313,21 +1315,21 @@ void DEMSolver::validateUserInputs() {
     //     LoadClumpType.");
     // }
 
-    // If not 2 GPUs detected, output warnings as needed
-    int ndevices = dTkT_GpuManager->getNumDevices();
-    if (ndevices == 0) {
-        DEME_ERROR(
-            "No GPU device is detected. Try lspci and see what you get.\nIf you indeed have GPU devices, maybe you "
-            "should try rebooting or reinstalling cuda components?");
-        // } else if (ndevices == 1) {
-        //     DEME_WARNING(
-        //         "One GPU device is detected. On consumer cards, DEME's performance edge is limited with only one"
-        //         "GPU.\nTry allocating 2 GPU devices if possible.");
-    } else if (ndevices > 2) {
-        DEME_WARNING(
-            "More than two GPU devices are detected.\nCurrently, DEME can make use of at most two devices.\nMore "
-            "devices will not improve the performance.");
-    }
+    // // If not 2 GPUs detected, output warnings as needed
+    // int ndevices = dTkT_GpuManager->getNumDevices();
+    // if (ndevices == 0) {
+    //     DEME_ERROR(
+    //         "No GPU device is detected. Try lspci and see what you get.\nIf you indeed have GPU devices, maybe you "
+    //         "should try rebooting or reinstalling cuda components?");
+    //     // } else if (ndevices == 1) {
+    //     //     DEME_WARNING(
+    //     //         "One GPU device is detected. On consumer cards, DEME's performance edge is limited with only one"
+    //     //         "GPU.\nTry allocating 2 GPU devices if possible.");
+    // } else if (ndevices > 2) {
+    //     DEME_WARNING(
+    //         "More than two GPU devices are detected.\nCurrently, DEME can make use of at most two devices.\nMore "
+    //         "devices will not improve the performance.");
+    // }
 
     // Box size OK?
     float3 user_box_size = m_user_box_max - m_user_box_min;

src/DEM/APIPublic.cpp

@@ -21,47 +21,120 @@
 namespace deme {
 
 DEMSolver::DEMSolver(unsigned int nGPUs) {
-    dTkT_InteractionManager = new ThreadManager();
-    kTMain_InteractionManager = new WorkerReportChannel();
-    dTMain_InteractionManager = new WorkerReportChannel();
+    if (nGPUs == 0) {
+        DEME_ERROR("DEMSolver was set to use 0 GPUs and that is currently not supported.");
+    }
 
-    // 2 means 2 threads (nGPUs is currently not used)
-    dTkT_GpuManager = new GpuManager(2);
+    dTkT_InteractionManager = std::make_unique<ThreadManager>();
+    kTMain_InteractionManager = std::make_unique<WorkerReportChannel>();
+    dTMain_InteractionManager = std::make_unique<WorkerReportChannel>();
 
     // Set default solver params
     setDefaultSolverParams();
 
+    // Determine which device IDs to use: scan available GPUs, use at most 2
+    int detected = GpuManager::scanNumDevices();
+    if (nGPUs > 2) {
+        DEME_WARNING("DEMSolver was requested to use %u GPUs, but at most 2 are supported. Using 2 GPUs.", nGPUs);
+        nGPUs = 2;
+    }
+    // Build a 2-element device ID list (one per thread); both threads share the same device when using 1 GPU
+    unsigned int nToUse = std::min((unsigned int)detected, nGPUs);
+    std::vector<int> device_ids;
+    if (nToUse >= 2) {
+        device_ids = {0, 1};
+    } else {
+        device_ids = {0, 0};
+    }
+    // Always use id list to avoid GpuManager deciding device usage by itself
+    dTkT_GpuManager = std::make_unique<GpuManager>(device_ids);
+
     // Thread-based worker creation may be needed as the workers allocate DualStructs on construction
     std::thread dT_construct([&]() {
         // Get a device/stream ID to use from the GPU Manager
         const GpuManager::StreamInfo dT_stream_info = dTkT_GpuManager->getAvailableStream();
         DEME_GPU_CALL(cudaSetDevice(dT_stream_info.device));
-        dT = new DEMDynamicThread(dTMain_InteractionManager, dTkT_InteractionManager, dT_stream_info);
+        dT = std::make_unique<DEMDynamicThread>(dTMain_InteractionManager.get(), dTkT_InteractionManager.get(),
+                                                dT_stream_info);
+    });
+
+    std::thread kT_construct([&]() {
+        const GpuManager::StreamInfo kT_stream_info = dTkT_GpuManager->getAvailableStream();
+        DEME_GPU_CALL(cudaSetDevice(kT_stream_info.device));
+        kT = std::make_unique<DEMKinematicThread>(kTMain_InteractionManager.get(), dTkT_InteractionManager.get(),
+                                                  kT_stream_info);
+    });
+
+    dT_construct.join();
+    kT_construct.join();
+
+    // Make friends
+    dT->kT = kT.get();
+    kT->dT = dT.get();
+}
+
+DEMSolver::DEMSolver(std::vector<int> device_ids) {
+    dTkT_InteractionManager = std::make_unique<ThreadManager>();
+    kTMain_InteractionManager = std::make_unique<WorkerReportChannel>();
+    dTMain_InteractionManager = std::make_unique<WorkerReportChannel>();
+
+    // Set default solver params
+    setDefaultSolverParams();
+
+    // Validate device IDs against the number of physically available GPUs
+    int detected = GpuManager::scanNumDevices();
+    if (device_ids.empty()) {
+        DEME_ERROR("DEMSolver was given an empty device ID list. Please provide at least one device ID.");
+    }
+    for (int id : device_ids) {
+        if (id < 0 || id >= detected) {
+            DEME_ERROR("DEMSolver was given device ID %d, but only %d GPU device(s) are available.", id, detected);
+        }
+    }
+    if (device_ids.size() > 2) {
+        DEME_WARNING("DEMSolver was given %zu device IDs, but at most 2 are supported. Using only the first 2.",
+                     device_ids.size());
+        device_ids.resize(2);
+    }
+    // When only one device is specified, both threads run on that device
+    if (device_ids.size() == 1) {
+        device_ids = {device_ids[0], device_ids[0]};
+    }
+    // Always use id list to avoid GpuManager deciding device usage by itself
+    dTkT_GpuManager = std::make_unique<GpuManager>(device_ids);
+
+    // Thread-based worker creation may be needed as the workers allocate DualStructs on construction
+    std::thread dT_construct([&]() {
+        const GpuManager::StreamInfo dT_stream_info = dTkT_GpuManager->getAvailableStream();
+        DEME_GPU_CALL(cudaSetDevice(dT_stream_info.device));
+        dT = std::make_unique<DEMDynamicThread>(dTMain_InteractionManager.get(), dTkT_InteractionManager.get(),
+                                                dT_stream_info);
     });
 
     std::thread kT_construct([&]() {
         const GpuManager::StreamInfo kT_stream_info = dTkT_GpuManager->getAvailableStream();
         DEME_GPU_CALL(cudaSetDevice(kT_stream_info.device));
-        kT = new DEMKinematicThread(kTMain_InteractionManager, dTkT_InteractionManager, kT_stream_info);
+        kT = std::make_unique<DEMKinematicThread>(kTMain_InteractionManager.get(), dTkT_InteractionManager.get(),
+                                                  kT_stream_info);
     });
 
     dT_construct.join();
     kT_construct.join();
 
     // Make friends
-    dT->kT = kT;
-    kT->dT = dT;
+    dT->kT = kT.get();
+    kT->dT = dT.get();
 }
 
 DEMSolver::~DEMSolver() {
     if (sys_initialized)
         DoDynamicsThenSync(0.0);
-    delete kT;
-    delete dT;
-    delete kTMain_InteractionManager;
-    delete dTMain_InteractionManager;
-    delete dTkT_InteractionManager;
-    delete dTkT_GpuManager;
+    kT.reset();
+    dT.reset();
+    kTMain_InteractionManager.reset();
+    dTMain_InteractionManager.reset();
+    dTkT_InteractionManager.reset();
+    dTkT_GpuManager.reset();
 }
 
 void DEMSolver::SetVerbosity(const std::string& verbose) {
@@ -1910,13 +1983,13 @@ std::shared_ptr<DEMMeshConnected> DEMSolver::AddWavefrontMeshObject(const std::s
 }
 
 std::shared_ptr<DEMInspector> DEMSolver::CreateInspector(const std::string& quantity) {
-    DEMInspector insp(this, this->dT, quantity);
+    DEMInspector insp(this, this->dT.get(), quantity);
     m_inspectors.push_back(std::make_shared<DEMInspector>(std::move(insp)));
     return m_inspectors.back();
 }
 
 std::shared_ptr<DEMInspector> DEMSolver::CreateInspector(const std::string& quantity, const std::string& region) {
-    DEMInspector insp(this, this->dT, quantity, region);
+    DEMInspector insp(this, this->dT.get(), quantity, region);
     m_inspectors.push_back(std::make_shared<DEMInspector>(std::move(insp)));
     return m_inspectors.back();
 }

src/DEM/dT.cpp

@@ -269,7 +269,7 @@ bodyID_t DEMDynamicThread::getGeoOwnerID(const bodyID_t& geoB, const contact_t&
 }
 
 // packTransferPointers
-void DEMDynamicThread::packTransferPointers(DEMKinematicThread*& kT) {
+void DEMDynamicThread::packTransferPointers(DEMKinematicThread* kT) {
     // These are the pointers for sending data to dT
     granData->pKTOwnedBuffer_absVel = kT->absVel_buffer.data();
     granData->pKTOwnedBuffer_voxelID = kT->voxelID_buffer.data();

src/DEM/dT.h

@@ -575,7 +575,7 @@ class DEMDynamicThread {
 
     /// Put sim data array pointers in place
     void packDataPointers();
-    void packTransferPointers(DEMKinematicThread*& kT);
+    void packTransferPointers(DEMKinematicThread* kT);
 
     // Move array data to or from device
     void migrateDataToDevice();

src/DEM/kT.cpp

@@ -518,7 +518,7 @@ void DEMKinematicThread::migrateDeviceModifiableInfoToHost() {
     migrateFamilyToHost();
 }
 
-void DEMKinematicThread::packTransferPointers(DEMDynamicThread*& dT) {
+void DEMKinematicThread::packTransferPointers(DEMDynamicThread* dT) {
     // Set the pointers to dT owned buffers
     granData->pDTOwnedBuffer_nContactPairs = &(dT->nContactPairs_buffer);
     granData->pDTOwnedBuffer_idGeometryA = dT->idGeometryA_buffer.data();

src/DEM/kT.h

@@ -342,7 +342,7 @@ class DEMKinematicThread {
 
     // Put sim data array pointers in place
     void packDataPointers();
-    void packTransferPointers(DEMDynamicThread*& dT);
+    void packTransferPointers(DEMDynamicThread* dT);
 
     // Move array data to or from device
     void migrateDataToDevice();

src/core/utils/GpuManager.cpp

@@ -1,4 +1,5 @@
 #include <algorithm>
+#include <set>
 #include <stdexcept>
 #include <iostream>
 
@@ -32,6 +33,24 @@ GpuManager::GpuManager(unsigned int total_streams) {
     }
 }
 
+GpuManager::GpuManager(const std::vector<int>& device_ids) {
+    // ndevices counts distinct physical devices in use
+    std::set<int> unique_ids(device_ids.begin(), device_ids.end());
+    ndevices = static_cast<int>(unique_ids.size());
+
+    if (!device_ids.empty()) {
+        int max_id = *std::max_element(device_ids.begin(), device_ids.end());
+        this->streams.resize(max_id + 1);
+    }
+
+    // Each entry in device_ids creates one stream; duplicate device IDs are intentional and result in multiple
+    // streams on the same device (used when both kT and dT threads run on the same physical device).
+    for (int dev_id : device_ids) {
+        cudaStream_t new_stream = nullptr;
+        this->streams[dev_id].push_back(StreamInfo{dev_id, new_stream, false});
+    }
+}
+
 // TODO: add CUDA error checking
 GpuManager::~GpuManager() {
     for (auto outer = this->streams.begin(); outer != this->streams.end(); outer++) {

src/core/utils/GpuManager.h

@@ -9,6 +9,8 @@
 class GpuManager {
   public:
     GpuManager(unsigned int total_streams = 1);
+    // Construct using explicit device IDs (one stream per device ID).
+    GpuManager(const std::vector<int>& device_ids);
     ~GpuManager();
 
     struct StreamInfo {
@@ -24,7 +26,7 @@ class GpuManager {
     // Returns the HIGHEST number of streams per device.
     unsigned int getMaxStreamsPerDevice();
 
-    int scanNumDevices();
+    static int scanNumDevices();
 
     // DO NOT USE UNLESS YOU INTEND TO MANUALLY HANDLE YOUR STREAMS.
     const std::vector<StreamInfo>& getStreamsFromDevice(int index);