Fix for multinode processing

Author: razdoburdin

plugin/sycl/device_manager.cc

@@ -19,20 +19,27 @@ ::sycl::queue* DeviceManager::GetQueue(const DeviceOrd& device_spec) const {
     size_t queue_idx;
     bool not_use_default_selector = (device_spec.ordinal != kDefaultOrdinal) ||
                                     (collective::IsDistributed());
-    DeviceRegister& device_register = GetDevicesRegister();
     if (not_use_default_selector) {
-        const int device_idx =
-            collective::IsDistributed() ? collective::GetRank() : device_spec.ordinal;
+        DeviceRegister& device_register = GetDevicesRegister();
         if (device_spec.IsSyclDefault()) {
             auto& devices = device_register.devices;
+            const int device_idx = collective::IsDistributed()
+                                   ? collective::GetRank() % devices.size()
+                                   : device_spec.ordinal;
             CHECK_LT(device_idx, devices.size());
             queue_idx = device_idx;
         } else if (device_spec.IsSyclCPU()) {
             auto& cpu_devices_idxes = device_register.cpu_devices_idxes;
+            const int device_idx = collective::IsDistributed()
+                                   ? collective::GetRank() % cpu_devices_idxes.size()
+                                   : device_spec.ordinal;
             CHECK_LT(device_idx, cpu_devices_idxes.size());
             queue_idx = cpu_devices_idxes[device_idx];
         } else if (device_spec.IsSyclGPU()) {
             auto& gpu_devices_idxes = device_register.gpu_devices_idxes;
+            const int device_idx = collective::IsDistributed()
+                                   ? collective::GetRank() % gpu_devices_idxes.size()
+                                   : device_spec.ordinal;
             CHECK_LT(device_idx, gpu_devices_idxes.size());
             queue_idx = gpu_devices_idxes[device_idx];
         } else {