[Bindless Image] Implement support for device to device copy

usm image to usm image copy

Signed-off-by: Zhang, Winston <[email protected]>

Author: winstonzhang-intel

source/adapters/level_zero/image.cpp

@@ -972,18 +972,60 @@ ur_result_t urBindlessImagesImageCopyExp(
                   WaitList.Length, WaitList.ZeEventList));
     }
   } else if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_DEVICE) {
-    ze_image_region_t DstRegion;
-    UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset,
-                                 &pCopyRegion->copyExtent, DstRegion));
-    ze_image_region_t SrcRegion;
-    UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset,
-                                 &pCopyRegion->copyExtent, SrcRegion));
-    auto *UrImageDst = static_cast<_ur_image *>(pDst);
-    auto *UrImageSrc = static_cast<const _ur_image *>(pSrc);
-    ZE2UR_CALL(zeCommandListAppendImageCopyRegion,
-               (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage,
-                &DstRegion, &SrcRegion, ZeEvent, WaitList.Length,
-                WaitList.ZeEventList));
+    if (pSrcImageDesc->rowPitch != 0 && pDstImageDesc->rowPitch != 0) {
+      // Copy from pitched USM memory to pitched USM memory
+      uint32_t SrcRowPitch = pSrcImageDesc->rowPitch;
+      uint32_t DstRowPitch = pDstImageDesc->rowPitch;
+      ze_copy_region_t ZeDstRegion = {(uint32_t)pCopyRegion->dstOffset.x,
+                                      (uint32_t)pCopyRegion->dstOffset.y,
+                                      (uint32_t)pCopyRegion->dstOffset.z,
+                                      DstRowPitch,
+                                      (uint32_t)pCopyRegion->copyExtent.height,
+                                      (uint32_t)pCopyRegion->copyExtent.depth};
+      uint32_t DstSlicePitch = 0;
+      uint32_t SrcSlicePitch = 0;
+      ze_copy_region_t ZeSrcRegion = {(uint32_t)pCopyRegion->srcOffset.x,
+                                      (uint32_t)pCopyRegion->srcOffset.y,
+                                      (uint32_t)pCopyRegion->srcOffset.z,
+                                      SrcRowPitch,
+                                      (uint32_t)pCopyRegion->copyExtent.height,
+                                      (uint32_t)pCopyRegion->copyExtent.depth};
+      ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion,
+                 (ZeCommandList, pDst, &ZeDstRegion, DstRowPitch, DstSlicePitch,
+                  pSrc, &ZeSrcRegion, SrcRowPitch, SrcSlicePitch, ZeEvent,
+                  WaitList.Length, WaitList.ZeEventList));
+    } else if (pSrcImageDesc->rowPitch == 0 && pDstImageDesc->rowPitch == 0) {
+      // Copy from Non-USM memory to Non-USM memory
+      ze_image_region_t DstRegion;
+      UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset,
+                                   &pCopyRegion->copyExtent, DstRegion));
+      ze_image_region_t SrcRegion;
+      UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset,
+                                   &pCopyRegion->copyExtent, SrcRegion));
+      auto *UrImageDst = static_cast<_ur_image *>(pDst);
+      auto *UrImageSrc = static_cast<const _ur_image *>(pSrc);
+      ZE2UR_CALL(zeCommandListAppendImageCopyRegion,
+                 (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage,
+                  &DstRegion, &SrcRegion, ZeEvent, WaitList.Length,
+                  WaitList.ZeEventList));
+
+    } else {
+      // Copy from Non-USM/pitched USM memory to pitched USM/Non-USM memory
+      // Note: This might be the same procedure as pitched USM to pitched USM
+      // Need further testing.
+      ze_image_region_t DstRegion;
+      UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset,
+                                   &pCopyRegion->copyExtent, DstRegion));
+      ze_image_region_t SrcRegion;
+      UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset,
+                                   &pCopyRegion->copyExtent, SrcRegion));
+      auto *UrImageDst = static_cast<_ur_image *>(pDst);
+      auto *UrImageSrc = static_cast<const _ur_image *>(pSrc);
+      ZE2UR_CALL(zeCommandListAppendImageCopyRegion,
+                 (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage,
+                  &DstRegion, &SrcRegion, ZeEvent, WaitList.Length,
+                  WaitList.ZeEventList));
+    }
   } else {
     logger::error("urBindlessImagesImageCopyExp: unexpected imageCopyFlags");
     return UR_RESULT_ERROR_UNSUPPORTED_FEATURE;