diff --git a/source/adapters/level_zero/image.cpp b/source/adapters/level_zero/image.cpp index 0a8fc1112d..9d7edd25dd 100644 --- a/source/adapters/level_zero/image.cpp +++ b/source/adapters/level_zero/image.cpp @@ -538,18 +538,60 @@ ur_result_t urBindlessImagesImageCopyExp( WaitList.Length, WaitList.ZeEventList)); } } else if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_DEVICE) { - ze_image_region_t DstRegion; - UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset, - &pCopyRegion->copyExtent, DstRegion)); - ze_image_region_t SrcRegion; - UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset, - &pCopyRegion->copyExtent, SrcRegion)); - auto *UrImageDst = static_cast<_ur_image *>(pDst); - auto *UrImageSrc = static_cast(pSrc); - ZE2UR_CALL(zeCommandListAppendImageCopyRegion, - (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage, - &DstRegion, &SrcRegion, ZeEvent, WaitList.Length, - WaitList.ZeEventList)); + if (pSrcImageDesc->rowPitch != 0 && pDstImageDesc->rowPitch != 0) { + // Copy from pitched USM memory to pitched USM memory + uint32_t SrcRowPitch = pSrcImageDesc->rowPitch; + uint32_t DstRowPitch = pDstImageDesc->rowPitch; + ze_copy_region_t ZeDstRegion = {(uint32_t)pCopyRegion->dstOffset.x, + (uint32_t)pCopyRegion->dstOffset.y, + (uint32_t)pCopyRegion->dstOffset.z, + DstRowPitch, + (uint32_t)pCopyRegion->copyExtent.height, + (uint32_t)pCopyRegion->copyExtent.depth}; + uint32_t DstSlicePitch = 0; + uint32_t SrcSlicePitch = 0; + ze_copy_region_t ZeSrcRegion = {(uint32_t)pCopyRegion->srcOffset.x, + (uint32_t)pCopyRegion->srcOffset.y, + (uint32_t)pCopyRegion->srcOffset.z, + SrcRowPitch, + (uint32_t)pCopyRegion->copyExtent.height, + (uint32_t)pCopyRegion->copyExtent.depth}; + ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, + (ZeCommandList, pDst, &ZeDstRegion, DstRowPitch, DstSlicePitch, + pSrc, &ZeSrcRegion, SrcRowPitch, SrcSlicePitch, ZeEvent, + WaitList.Length, WaitList.ZeEventList)); + } else if (pSrcImageDesc->rowPitch == 0 && pDstImageDesc->rowPitch == 0) { + // Copy from Non-USM memory to Non-USM memory + ze_image_region_t DstRegion; + UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset, + &pCopyRegion->copyExtent, DstRegion)); + ze_image_region_t SrcRegion; + UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset, + &pCopyRegion->copyExtent, SrcRegion)); + auto *UrImageDst = static_cast<_ur_image *>(pDst); + auto *UrImageSrc = static_cast(pSrc); + ZE2UR_CALL(zeCommandListAppendImageCopyRegion, + (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage, + &DstRegion, &SrcRegion, ZeEvent, WaitList.Length, + WaitList.ZeEventList)); + + } else { + // Copy from Non-USM/pitched USM memory to pitched USM/Non-USM memory + // Note: This might be the same procedure as pitched USM to + // pitched USM. Need further testing. + ze_image_region_t DstRegion; + UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset, + &pCopyRegion->copyExtent, DstRegion)); + ze_image_region_t SrcRegion; + UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset, + &pCopyRegion->copyExtent, SrcRegion)); + auto *UrImageDst = static_cast<_ur_image *>(pDst); + auto *UrImageSrc = static_cast(pSrc); + ZE2UR_CALL(zeCommandListAppendImageCopyRegion, + (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage, + &DstRegion, &SrcRegion, ZeEvent, WaitList.Length, + WaitList.ZeEventList)); + } } else { logger::error("urBindlessImagesImageCopyExp: unexpected imageCopyFlags"); return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; @@ -696,7 +738,6 @@ ur_result_t urBindlessImagesMapExternalArrayExp( const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, ur_exp_external_mem_handle_t hExternalMem, ur_exp_image_mem_native_handle_t *phImageMem) { - UR_ASSERT(hContext && hDevice && hExternalMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); UR_ASSERT(pImageFormat && pImageDesc, UR_RESULT_ERROR_INVALID_NULL_POINTER);