Merge branch 'main' into sanitizer-pr-cpu-local

Author: AllanZyne

.github/workflows/cmake.yml

@@ -68,7 +68,7 @@ jobs:
       if: matrix.os == 'ubuntu-22.04'
       run: |
         sudo apt install libncurses5
-        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/nightly-2023-09-21/sycl_linux.tar.gz
+        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/nightly-2024-01-29/sycl_linux.tar.gz
         mkdir -p ${{github.workspace}}/dpcpp_compiler
         tar -xvf ${{github.workspace}}/dpcpp_compiler.tar.gz -C ${{github.workspace}}/dpcpp_compiler
 
@@ -169,7 +169,8 @@ jobs:
           {name: CUDA, platform: ""},
           {name: HIP, platform: ""},
           {name: L0, platform: ""},
-          {name: OPENCL, platform: "Intel(R) OpenCL"}
+          {name: OPENCL, platform: "Intel(R) OpenCL"},
+          {name: NATIVE_CPU, platform: ""}
         ]
         build_type: [Debug, Release]
         compiler: [{c: gcc, cxx: g++}, {c: clang, cxx: clang++}]
@@ -184,7 +185,7 @@ jobs:
 
     - name: Download DPC++
       run: |
-        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/nightly-2023-09-21/sycl_linux.tar.gz
+        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/nightly-2024-01-29/sycl_linux.tar.gz
         mkdir dpcpp_compiler
         tar -xvf ${{github.workspace}}/dpcpp_compiler.tar.gz -C dpcpp_compiler
 
@@ -230,7 +231,8 @@ jobs:
 
   examples-build-hw:
     name: Build - examples on HW
-    if: github.repository == 'oneapi-src/unified-runtime'  # run only on upstream; forks won't have the HW
+    # if: github.repository == 'oneapi-src/unified-runtime'  # run only on upstream; forks won't have the HW
+    if: false  # temporaily disabled due to conda env setup issues
     strategy:
       matrix:
         adapter: [

.github/workflows/nightly.yml

@@ -26,7 +26,7 @@ jobs:
 
     - name: Download DPC++
       run: |
-        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/nightly-2023-08-31/sycl_linux.tar.gz
+        wget -O ${{github.workspace}}/dpcpp_compiler.tar.gz https://github.com/intel/llvm/releases/download/nightly-2024-01-29/sycl_linux.tar.gz
         mkdir dpcpp_compiler
         tar -xvf ${{github.workspace}}/dpcpp_compiler.tar.gz -C dpcpp_compiler
 

cmake/FindRocmAgentEnumerator.cmake

@@ -9,7 +9,7 @@
 #                                  rocm_agent_enumerator is found.
 #
 
-find_program(ROCM_AGENT_ENUMERATOR NAMES not_rocm_agent_enumerator)
+find_program(ROCM_AGENT_ENUMERATOR NAMES rocm_agent_enumerator)
 
 if(ROCM_AGENT_ENUMERATOR)
     set(ROCM_AGENT_ENUMERATOR_FOUND TRUE)

include/ur_api.h

@@ -2723,13 +2723,15 @@ urMemBufferPartition(
 ///     - ::UR_RESULT_ERROR_ADAPTER_SPECIFIC
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_HANDLE
 ///         + `NULL == hMem`
+///         + `NULL == hDevice`
 ///     - ::UR_RESULT_ERROR_INVALID_NULL_POINTER
 ///         + `NULL == phNativeMem`
 ///     - ::UR_RESULT_ERROR_UNSUPPORTED_FEATURE
 ///         + If the adapter has no underlying equivalent handle.
 UR_APIEXPORT ur_result_t UR_APICALL
 urMemGetNativeHandle(
     ur_mem_handle_t hMem,           ///< [in] handle of the mem.
+    ur_device_handle_t hDevice,     ///< [in] handle of the device that the native handle will be resident on.
     ur_native_handle_t *phNativeMem ///< [out] a pointer to the native handle of the mem.
 );
 
@@ -9488,6 +9490,7 @@ typedef struct ur_mem_buffer_partition_params_t {
 ///     allowing the callback the ability to modify the parameter's value
 typedef struct ur_mem_get_native_handle_params_t {
     ur_mem_handle_t *phMem;
+    ur_device_handle_t *phDevice;
     ur_native_handle_t **pphNativeMem;
 } ur_mem_get_native_handle_params_t;
 

include/ur_ddi.h

@@ -770,6 +770,7 @@ typedef ur_result_t(UR_APICALL *ur_pfnMemBufferPartition_t)(
 /// @brief Function-pointer for urMemGetNativeHandle
 typedef ur_result_t(UR_APICALL *ur_pfnMemGetNativeHandle_t)(
     ur_mem_handle_t,
+    ur_device_handle_t,
     ur_native_handle_t *);
 
 ///////////////////////////////////////////////////////////////////////////////

include/ur_print.hpp

@@ -11174,6 +11174,12 @@ inline std::ostream &operator<<(std::ostream &os, [[maybe_unused]] const struct
     ur::details::printPtr(os,
                           *(params->phMem));
 
+    os << ", ";
+    os << ".hDevice = ";
+
+    ur::details::printPtr(os,
+                          *(params->phDevice));
+
     os << ", ";
     os << ".phNativeMem = ";
 

scripts/core/memory.yml

@@ -432,6 +432,10 @@ params:
       name: hMem
       desc: |
             [in] handle of the mem.
+    - type: $x_device_handle_t
+      name: hDevice
+      desc: |
+            [in] handle of the device that the native handle will be resident on.
     - type: $x_native_handle_t*
       name: phNativeMem
       desc: |

source/adapters/cuda/enqueue.cpp

@@ -862,7 +862,7 @@ static size_t imageElementByteSize(CUDA_ARRAY_DESCRIPTOR ArrayDesc) {
   }
 }
 
-/// General ND memory copy operation for images (where N > 1).
+/// General ND memory copy operation for images.
 /// This function requires the corresponding CUDA context to be at the top of
 /// the context stack
 /// If the source and/or destination is an array, SrcPtr and/or DstPtr
@@ -877,27 +877,27 @@ static ur_result_t commonEnqueueMemImageNDCopy(
   UR_ASSERT(DstType == CU_MEMORYTYPE_ARRAY || DstType == CU_MEMORYTYPE_HOST,
             UR_RESULT_ERROR_INVALID_MEM_OBJECT);
 
-  if (ImgType == UR_MEM_TYPE_IMAGE2D) {
+  if (ImgType == UR_MEM_TYPE_IMAGE1D || ImgType == UR_MEM_TYPE_IMAGE2D) {
     CUDA_MEMCPY2D CpyDesc;
     memset(&CpyDesc, 0, sizeof(CpyDesc));
     CpyDesc.srcMemoryType = SrcType;
     if (SrcType == CU_MEMORYTYPE_ARRAY) {
       CpyDesc.srcArray = *static_cast<const CUarray *>(SrcPtr);
       CpyDesc.srcXInBytes = SrcOffset.x;
-      CpyDesc.srcY = SrcOffset.y;
+      CpyDesc.srcY = (ImgType == UR_MEM_TYPE_IMAGE1D) ? 0 : SrcOffset.y;
     } else {
       CpyDesc.srcHost = SrcPtr;
     }
     CpyDesc.dstMemoryType = DstType;
     if (DstType == CU_MEMORYTYPE_ARRAY) {
       CpyDesc.dstArray = *static_cast<CUarray *>(DstPtr);
       CpyDesc.dstXInBytes = DstOffset.x;
-      CpyDesc.dstY = DstOffset.y;
+      CpyDesc.dstY = (ImgType == UR_MEM_TYPE_IMAGE1D) ? 0 : DstOffset.y;
     } else {
       CpyDesc.dstHost = DstPtr;
     }
     CpyDesc.WidthInBytes = Region.width;
-    CpyDesc.Height = Region.height;
+    CpyDesc.Height = (ImgType == UR_MEM_TYPE_IMAGE1D) ? 1 : Region.height;
     UR_CHECK_ERROR(cuMemcpy2DAsync(&CpyDesc, CuStream));
     return UR_RESULT_SUCCESS;
   }
@@ -1124,21 +1124,17 @@ UR_APIEXPORT ur_result_t UR_APICALL urEnqueueMemImageCopy(
     }
 
     ur_mem_type_t ImgType = std::get<SurfaceMem>(hImageSrc->Mem).getImageType();
-    if (ImgType == UR_MEM_TYPE_IMAGE1D) {
-      UR_CHECK_ERROR(cuMemcpyAtoA(DstArray, DstByteOffsetX, SrcArray,
-                                  SrcByteOffsetX, BytesToCopy));
-    } else {
-      ur_rect_region_t AdjustedRegion = {BytesToCopy, region.height,
-                                         region.depth};
-      ur_rect_offset_t SrcOffset = {SrcByteOffsetX, srcOrigin.y, srcOrigin.z};
-      ur_rect_offset_t DstOffset = {DstByteOffsetX, dstOrigin.y, dstOrigin.z};
 
-      Result = commonEnqueueMemImageNDCopy(
-          CuStream, ImgType, AdjustedRegion, &SrcArray, CU_MEMORYTYPE_ARRAY,
-          SrcOffset, &DstArray, CU_MEMORYTYPE_ARRAY, DstOffset);
-      if (Result != UR_RESULT_SUCCESS) {
-        return Result;
-      }
+    ur_rect_region_t AdjustedRegion = {BytesToCopy, region.height,
+                                       region.depth};
+    ur_rect_offset_t SrcOffset = {SrcByteOffsetX, srcOrigin.y, srcOrigin.z};
+    ur_rect_offset_t DstOffset = {DstByteOffsetX, dstOrigin.y, dstOrigin.z};
+
+    Result = commonEnqueueMemImageNDCopy(
+        CuStream, ImgType, AdjustedRegion, &SrcArray, CU_MEMORYTYPE_ARRAY,
+        SrcOffset, &DstArray, CU_MEMORYTYPE_ARRAY, DstOffset);
+    if (Result != UR_RESULT_SUCCESS) {
+      return Result;
     }
 
     if (phEvent) {

source/adapters/cuda/image.cpp

@@ -239,29 +239,38 @@ ur_result_t urTextureCreate(ur_sampler_handle_t hSampler,
 
   try {
     /// pi_sampler_properties
+    /// Layout of UR samplers for CUDA
+    ///
+    /// Sampler property layout:
     /// |     <bits>     | <usage>
     /// -----------------------------------
-    /// |  31 30 ... 6   | N/A
-    /// |       5        | mip filter mode
-    /// |     4 3 2      | addressing mode
+    /// |  31 30 ... 12  | N/A
+    /// |       11       | mip filter mode
+    /// |    10 9 8      | addressing mode 3
+    /// |     7 6 5      | addressing mode 2
+    /// |     4 3 2      | addressing mode 1
     /// |       1        | filter mode
     /// |       0        | normalize coords
     CUDA_TEXTURE_DESC ImageTexDesc = {};
-    CUaddress_mode AddrMode = {};
-    ur_sampler_addressing_mode_t AddrModeProp = hSampler->getAddressingMode();
-    if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE -
-                         UR_SAMPLER_ADDRESSING_MODE_NONE)) {
-      AddrMode = CU_TR_ADDRESS_MODE_CLAMP;
-    } else if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_CLAMP -
-                                UR_SAMPLER_ADDRESSING_MODE_NONE)) {
-      AddrMode = CU_TR_ADDRESS_MODE_BORDER;
-    } else if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_REPEAT -
-                                UR_SAMPLER_ADDRESSING_MODE_NONE)) {
-      AddrMode = CU_TR_ADDRESS_MODE_WRAP;
-    } else if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_MIRRORED_REPEAT -
-                                UR_SAMPLER_ADDRESSING_MODE_NONE)) {
-      AddrMode = CU_TR_ADDRESS_MODE_MIRROR;
+    CUaddress_mode AddrMode[3];
+    for (size_t i = 0; i < 3; i++) {
+      ur_sampler_addressing_mode_t AddrModeProp =
+          hSampler->getAddressingModeDim(i);
+      if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_CLAMP_TO_EDGE -
+                           UR_SAMPLER_ADDRESSING_MODE_NONE)) {
+        AddrMode[i] = CU_TR_ADDRESS_MODE_CLAMP;
+      } else if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_CLAMP -
+                                  UR_SAMPLER_ADDRESSING_MODE_NONE)) {
+        AddrMode[i] = CU_TR_ADDRESS_MODE_BORDER;
+      } else if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_REPEAT -
+                                  UR_SAMPLER_ADDRESSING_MODE_NONE)) {
+        AddrMode[i] = CU_TR_ADDRESS_MODE_WRAP;
+      } else if (AddrModeProp == (UR_SAMPLER_ADDRESSING_MODE_MIRRORED_REPEAT -
+                                  UR_SAMPLER_ADDRESSING_MODE_NONE)) {
+        AddrMode[i] = CU_TR_ADDRESS_MODE_MIRROR;
+      }
     }
+
     CUfilter_mode FilterMode;
     ur_sampler_filter_mode_t FilterModeProp = hSampler->getFilterMode();
     FilterMode =
@@ -278,14 +287,15 @@ ur_result_t urTextureCreate(ur_sampler_handle_t hSampler,
     ImageTexDesc.minMipmapLevelClamp = hSampler->MinMipmapLevelClamp;
     ImageTexDesc.maxAnisotropy = static_cast<unsigned>(hSampler->MaxAnisotropy);
 
-    // The address modes can interfere with other dimensionsenqueueEventsWait
+    // The address modes can interfere with other dimensions
     // e.g. 1D texture sampling can be interfered with when setting other
     // dimension address modes despite their nonexistence.
-    ImageTexDesc.addressMode[0] = AddrMode; // 1D
-    ImageTexDesc.addressMode[1] =
-        pImageDesc->height > 0 ? AddrMode : ImageTexDesc.addressMode[1]; // 2D
+    ImageTexDesc.addressMode[0] = AddrMode[0]; // 1D
+    ImageTexDesc.addressMode[1] = pImageDesc->height > 0
+                                      ? AddrMode[1]
+                                      : ImageTexDesc.addressMode[1]; // 2D
     ImageTexDesc.addressMode[2] =
-        pImageDesc->depth > 0 ? AddrMode : ImageTexDesc.addressMode[2]; // 3D
+        pImageDesc->depth > 0 ? AddrMode[2] : ImageTexDesc.addressMode[2]; // 3D
 
     // flags takes the normalized coordinates setting -- unnormalized is default
     ImageTexDesc.flags = (hSampler->isNormalizedCoords())

source/adapters/cuda/memory.cpp

@@ -161,8 +161,8 @@ UR_APIEXPORT ur_result_t UR_APICALL urMemRelease(ur_mem_handle_t hMem) {
 /// \param[out] phNativeMem Set to the native handle of the UR mem object.
 ///
 /// \return UR_RESULT_SUCCESS
-UR_APIEXPORT ur_result_t UR_APICALL
-urMemGetNativeHandle(ur_mem_handle_t hMem, ur_native_handle_t *phNativeMem) {
+UR_APIEXPORT ur_result_t UR_APICALL urMemGetNativeHandle(
+    ur_mem_handle_t hMem, ur_device_handle_t, ur_native_handle_t *phNativeMem) {
   *phNativeMem = reinterpret_cast<ur_native_handle_t>(
       std::get<BufferMem>(hMem->Mem).get());
   return UR_RESULT_SUCCESS;