diff --git a/sycl/test-e2e/bindless_images/lit.local.cfg b/sycl/test-e2e/bindless_images/lit.local.cfg index cfd5e0241ca89..6c83825cc1ac0 100644 --- a/sycl/test-e2e/bindless_images/lit.local.cfg +++ b/sycl/test-e2e/bindless_images/lit.local.cfg @@ -1,5 +1,3 @@ # Support of this feature is waiting for the upstream PR # https://github.com/llvm/llvm-project/pull/127737 to be merged with intel/llvm. config.unsupported_features += ['spirv-backend'] -# V2 adapter does not support bindless images yet -config.unsupported_features += ['level_zero_v2_adapter'] diff --git a/unified-runtime/source/adapters/level_zero/CMakeLists.txt b/unified-runtime/source/adapters/level_zero/CMakeLists.txt index 6eb08a6d93dde..9f819b09ce811 100644 --- a/unified-runtime/source/adapters/level_zero/CMakeLists.txt +++ b/unified-runtime/source/adapters/level_zero/CMakeLists.txt @@ -20,6 +20,7 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/context.hpp ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp ${CMAKE_CURRENT_SOURCE_DIR}/event.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/usm.hpp ${CMAKE_CURRENT_SOURCE_DIR}/memory.hpp ${CMAKE_CURRENT_SOURCE_DIR}/kernel.hpp @@ -28,7 +29,6 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp ${CMAKE_CURRENT_SOURCE_DIR}/queue.hpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/ur_level_zero.cpp @@ -48,7 +48,7 @@ if(UR_BUILD_ADAPTER_L0) ${CMAKE_CURRENT_SOURCE_DIR}/queue.cpp ${CMAKE_CURRENT_SOURCE_DIR}/sampler.cpp ${CMAKE_CURRENT_SOURCE_DIR}/image.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/../../ur/ur.cpp @@ -125,20 +125,20 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/adapter.hpp ${CMAKE_CURRENT_SOURCE_DIR}/common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/device.hpp + ${CMAKE_CURRENT_SOURCE_DIR}/image_common.hpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.hpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.hpp ${CMAKE_CURRENT_SOURCE_DIR}/program.hpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.hpp ${CMAKE_CURRENT_SOURCE_DIR}/adapter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/device.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/image_common.cpp ${CMAKE_CURRENT_SOURCE_DIR}/ur_interface_loader.cpp ${CMAKE_CURRENT_SOURCE_DIR}/platform.cpp ${CMAKE_CURRENT_SOURCE_DIR}/physical_mem.cpp ${CMAKE_CURRENT_SOURCE_DIR}/program.cpp - ${CMAKE_CURRENT_SOURCE_DIR}/helpers/image_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/kernel_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/helpers/memory_helpers.cpp ${CMAKE_CURRENT_SOURCE_DIR}/usm_p2p.cpp @@ -173,6 +173,7 @@ if(UR_BUILD_ADAPTER_L0_V2) ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_provider_counter.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/event_provider_normal.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/event.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/v2/image.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/kernel.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/memory.cpp ${CMAKE_CURRENT_SOURCE_DIR}/v2/queue_api.cpp diff --git a/unified-runtime/source/adapters/level_zero/helpers/image_helpers.cpp b/unified-runtime/source/adapters/level_zero/helpers/image_helpers.cpp deleted file mode 100644 index 9da1750e8c1f9..0000000000000 --- a/unified-runtime/source/adapters/level_zero/helpers/image_helpers.cpp +++ /dev/null @@ -1,522 +0,0 @@ -//===--------- image_helpers.cpp - Level Zero Adapter --------------------===// -// -// Copyright (C) 2024 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// - -#include "image_helpers.hpp" - -/// Construct UR image format from ZE image desc. -ur_result_t ze2urImageFormat(const ze_image_desc_t *ZeImageDesc, - ur_image_format_t *UrImageFormat) { - const ze_image_format_t &ZeImageFormat = ZeImageDesc->format; - size_t ZeImageFormatTypeSize; - switch (ZeImageFormat.layout) { - case ZE_IMAGE_FORMAT_LAYOUT_8: - case ZE_IMAGE_FORMAT_LAYOUT_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: - ZeImageFormatTypeSize = 8; - break; - case ZE_IMAGE_FORMAT_LAYOUT_16: - case ZE_IMAGE_FORMAT_LAYOUT_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: - ZeImageFormatTypeSize = 16; - break; - case ZE_IMAGE_FORMAT_LAYOUT_32: - case ZE_IMAGE_FORMAT_LAYOUT_32_32: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: - ZeImageFormatTypeSize = 32; - break; - default: - logger::error( - "ze2urImageFormat: unsupported image format layout: layout = {}", - ZeImageFormat.layout); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - ur_image_channel_order_t ChannelOrder; - switch (ZeImageFormat.layout) { - case ZE_IMAGE_FORMAT_LAYOUT_8: - case ZE_IMAGE_FORMAT_LAYOUT_16: - case ZE_IMAGE_FORMAT_LAYOUT_32: - switch (ZeImageFormat.x) { - case ZE_IMAGE_FORMAT_SWIZZLE_R: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_R; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_A: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_A; - break; - default: - logger::error( - "ze2urImageFormat: unexpected image format channel x: x = {}", - ZeImageFormat.x); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_LAYOUT_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_32_32: - if (ZeImageFormat.x != ZE_IMAGE_FORMAT_SWIZZLE_R) { - logger::error( - "ze2urImageFormat: unexpected image format channel x: x = {}", - ZeImageFormat.x); - return UR_RESULT_ERROR_INVALID_VALUE; - } - switch (ZeImageFormat.y) { - case ZE_IMAGE_FORMAT_SWIZZLE_G: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RG; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_A: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RA; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_X: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RX; - break; - default: - logger::error( - "ze2urImageFormat: unexpected image format channel y: y = {}\n", - ZeImageFormat.y); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: - if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G) { - switch (ZeImageFormat.z) { - case ZE_IMAGE_FORMAT_SWIZZLE_B: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGB; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_X: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGX; - break; - default: - logger::error( - "ze2urImageFormat: unexpected image format channel z: z = {}\n", - ZeImageFormat.z); - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else { - logger::error("ze2urImageFormat: unexpected image format channel"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: - case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: - case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: - if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && - ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_B) { - switch (ZeImageFormat.w) { - case ZE_IMAGE_FORMAT_SWIZZLE_X: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBX; - break; - case ZE_IMAGE_FORMAT_SWIZZLE_A: - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBA; - break; - default: - logger::error("ze2urImageFormat: unexpected image format channel w: w " - "= {}", - ZeImageFormat.x); - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_A && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_G && - ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_B) { - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_ARGB; - } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_B && - ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && - ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_R && - ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_A) { - ChannelOrder = UR_IMAGE_CHANNEL_ORDER_BGRA; - } else { - logger::error("ze2urImageFormat: unexpected image format channel"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - default: - logger::error( - "ze2urImageFormat: unsupported image format layout: layout = {}", - ZeImageFormat.layout); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - ur_image_channel_type_t ChannelType; - switch (ZeImageFormat.type) { - case ZE_IMAGE_FORMAT_TYPE_UINT: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16; - break; - case 32: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_SINT: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16; - break; - case 32: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_UNORM: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT16; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_SNORM: - switch (ZeImageFormatTypeSize) { - case 8: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT8; - break; - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT16; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - case ZE_IMAGE_FORMAT_TYPE_FLOAT: - switch (ZeImageFormatTypeSize) { - case 16: - ChannelType = UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT; - break; - case 32: - ChannelType = UR_IMAGE_CHANNEL_TYPE_FLOAT; - break; - default: - logger::error("ze2urImageFormat: unexpected image format type size: size " - "= {}", - ZeImageFormatTypeSize); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - default: - logger::error("ze2urImageFormat: unsupported image format type: type = {}", - ZeImageFormat.type); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - UrImageFormat->channelOrder = ChannelOrder; - UrImageFormat->channelType = ChannelType; - return UR_RESULT_SUCCESS; -} - -bool Is3ChannelOrder(ur_image_channel_order_t ChannelOrder) { - switch (ChannelOrder) { - case UR_IMAGE_CHANNEL_ORDER_RGB: - case UR_IMAGE_CHANNEL_ORDER_RGX: - return true; - default: - return false; - } -} - -/// Construct ZE image desc from UR image format and desc. -ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, - const ur_image_desc_t *ImageDesc, - ZeStruct &ZeImageDesc) { - - auto [ZeImageFormatType, ZeImageFormatTypeSize] = - getImageFormatTypeAndSize(ImageFormat); - - if (ZeImageFormatTypeSize == 0) { - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - } - - // TODO: populate the layout mapping - ze_image_format_layout_t ZeImageFormatLayout; - switch (ImageFormat->channelOrder) { - case UR_IMAGE_CHANNEL_ORDER_RGBA: { - switch (ZeImageFormatTypeSize) { - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; - break; - default: - logger::error("ur2zeImageDesc: unexpected data type Size\n"); - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - } - break; - } - default: - logger::error("format layout = {}", ImageFormat->channelOrder); - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - break; - } - - ze_image_format_t ZeFormatDesc = { - ZeImageFormatLayout, ZeImageFormatType, - // TODO: are swizzles deducted from image_format->image_channel_order? - ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, - ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_A}; - - ze_image_type_t ZeImageType; - switch (ImageDesc->type) { - case UR_MEM_TYPE_IMAGE1D: - ZeImageType = ZE_IMAGE_TYPE_1D; - break; - case UR_MEM_TYPE_IMAGE2D: - ZeImageType = ZE_IMAGE_TYPE_2D; - break; - case UR_MEM_TYPE_IMAGE3D: - ZeImageType = ZE_IMAGE_TYPE_3D; - break; - case UR_MEM_TYPE_IMAGE1D_ARRAY: - ZeImageType = ZE_IMAGE_TYPE_1DARRAY; - break; - case UR_MEM_TYPE_IMAGE2D_ARRAY: - ZeImageType = ZE_IMAGE_TYPE_2DARRAY; - break; - default: - logger::error("ur2zeImageDesc: unsupported image type"); - return UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR; - } - - ZeImageDesc.arraylevels = ZeImageDesc.flags = 0; - ZeImageDesc.type = ZeImageType; - ZeImageDesc.format = ZeFormatDesc; - ZeImageDesc.width = ur_cast(ImageDesc->width); - ZeImageDesc.height = ur_cast(ImageDesc->height); - ZeImageDesc.depth = ur_cast(ImageDesc->depth); - ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); - ZeImageDesc.miplevels = ImageDesc->numMipLevel; - - return UR_RESULT_SUCCESS; -} - -/// Return element size in bytes of a pixel. -uint32_t getPixelSizeBytes(const ur_image_format_t *Format) { - uint32_t NumChannels = 0; - switch (Format->channelOrder) { - case UR_IMAGE_CHANNEL_ORDER_A: - case UR_IMAGE_CHANNEL_ORDER_R: - case UR_IMAGE_CHANNEL_ORDER_INTENSITY: - case UR_IMAGE_CHANNEL_ORDER_LUMINANCE: - case UR_IMAGE_CHANNEL_ORDER_FORCE_UINT32: - NumChannels = 1; - break; - case UR_IMAGE_CHANNEL_ORDER_RG: - case UR_IMAGE_CHANNEL_ORDER_RA: - case UR_IMAGE_CHANNEL_ORDER_RX: - NumChannels = 2; - break; - case UR_IMAGE_CHANNEL_ORDER_RGB: - case UR_IMAGE_CHANNEL_ORDER_RGX: - NumChannels = 3; - break; - case UR_IMAGE_CHANNEL_ORDER_RGBA: - case UR_IMAGE_CHANNEL_ORDER_BGRA: - case UR_IMAGE_CHANNEL_ORDER_ARGB: - case UR_IMAGE_CHANNEL_ORDER_ABGR: - case UR_IMAGE_CHANNEL_ORDER_RGBX: - case UR_IMAGE_CHANNEL_ORDER_SRGBA: - NumChannels = 4; - break; - default: - ur::unreachable(); - } - uint32_t ChannelTypeSizeInBytes = 0; - switch (Format->channelType) { - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: - ChannelTypeSizeInBytes = 1; - break; - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: - case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565: - case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555: - case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: - ChannelTypeSizeInBytes = 2; - break; - case UR_IMAGE_CHANNEL_TYPE_INT_101010: - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: - case UR_IMAGE_CHANNEL_TYPE_FLOAT: - case UR_IMAGE_CHANNEL_TYPE_FORCE_UINT32: - ChannelTypeSizeInBytes = 4; - break; - default: - ur::unreachable(); - } - return NumChannels * ChannelTypeSizeInBytes; -} - -ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, - ur_rect_offset_t *Origin, - ur_rect_region_t *Region, - ze_image_region_t &ZeRegion) { - UR_ASSERT(Origin, UR_RESULT_ERROR_INVALID_VALUE); - UR_ASSERT(Region, UR_RESULT_ERROR_INVALID_VALUE); - - if (ZeImageDesc.type == ZE_IMAGE_TYPE_1D || - ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) { - Region->height = 1; - Region->depth = 1; - } else if (ZeImageDesc.type == ZE_IMAGE_TYPE_2D || - ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) { - Region->depth = 1; - } - -#ifndef NDEBUG - UR_ASSERT((ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Origin->y == 0 && - Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Origin->z == 0) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), - UR_RESULT_ERROR_INVALID_VALUE); - - UR_ASSERT(Region->width && Region->height && Region->depth, - UR_RESULT_ERROR_INVALID_VALUE); - UR_ASSERT( - (ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Region->height == 1 && - Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Region->depth == 1) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || - (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), - UR_RESULT_ERROR_INVALID_VALUE); -#endif // !NDEBUG - - uint32_t OriginX = ur_cast(Origin->x); - uint32_t OriginY = ur_cast(Origin->y); - uint32_t OriginZ = ur_cast(Origin->z); - - uint32_t Width = ur_cast(Region->width); - uint32_t Height = (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) - ? ZeImageDesc.arraylevels - : ur_cast(Region->height); - uint32_t Depth = (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) - ? ZeImageDesc.arraylevels - : ur_cast(Region->depth); - - ZeRegion = {OriginX, OriginY, OriginZ, Width, Height, Depth}; - - return UR_RESULT_SUCCESS; -} - -std::pair -getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat) { - ze_image_format_type_t ZeImageFormatType; - size_t ZeImageFormatTypeSize; - switch (ImageFormat->channelType) { - case UR_IMAGE_CHANNEL_TYPE_FLOAT: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 32; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; - ZeImageFormatTypeSize = 8; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; - ZeImageFormatTypeSize = 16; - break; - } - case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: { - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; - ZeImageFormatTypeSize = 8; - break; - } - default: - logger::error("ur2zeImageDesc: unsupported image data type: data type = {}", - ImageFormat->channelType); - ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FORCE_UINT32; - ZeImageFormatTypeSize = 0; - } - return {ZeImageFormatType, ZeImageFormatTypeSize}; -} diff --git a/unified-runtime/source/adapters/level_zero/helpers/image_helpers.hpp b/unified-runtime/source/adapters/level_zero/helpers/image_helpers.hpp deleted file mode 100644 index 04c4e08d1b9d1..0000000000000 --- a/unified-runtime/source/adapters/level_zero/helpers/image_helpers.hpp +++ /dev/null @@ -1,38 +0,0 @@ -//===--------- image_helpers.hpp - Level Zero Adapter --------------------===// -// -// Copyright (C) 2024 Intel Corporation -// -// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM -// Exceptions. See LICENSE.TXT -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -#pragma once - -#include -#include -#include - -#include "../common.hpp" - -/// Construct UR image format from ZE image desc. -ur_result_t ze2urImageFormat(const ze_image_desc_t *ZeImageDesc, - ur_image_format_t *UrImageFormat); - -/// Construct ZE image desc from UR image format and desc. -ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, - const ur_image_desc_t *ImageDesc, - ZeStruct &ZeImageDesc); - -/// Return element size in bytes of a pixel. -uint32_t getPixelSizeBytes(const ur_image_format_t *Format); - -bool Is3ChannelOrder(ur_image_channel_order_t ChannelOrder); - -ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, - ur_rect_offset_t *Origin, - ur_rect_region_t *Region, - ze_image_region_t &ZeRegion); - -std::pair -getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat); diff --git a/unified-runtime/source/adapters/level_zero/image.cpp b/unified-runtime/source/adapters/level_zero/image.cpp index 5bf70d209e520..7770c7d2126c4 100644 --- a/unified-runtime/source/adapters/level_zero/image.cpp +++ b/unified-runtime/source/adapters/level_zero/image.cpp @@ -1,6 +1,6 @@ //===--------- image.cpp - Level Zero Adapter -----------------------------===// // -// Copyright (C) 2023 Intel Corporation +// Copyright (C) 2023-2025 Intel Corporation // // Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM // Exceptions. See LICENSE.TXT @@ -11,357 +11,30 @@ #include "common.hpp" #include "context.hpp" #include "event.hpp" -#include "helpers/image_helpers.hpp" +#include "image_common.hpp" #include "logger/ur_logger.hpp" +#include "memory.hpp" #include "sampler.hpp" #include "ur_interface_loader.hpp" -#include "ur_level_zero.hpp" -typedef ze_result_t(ZE_APICALL *zeImageGetDeviceOffsetExp_pfn)( - ze_image_handle_t hImage, uint64_t *pDeviceOffset); - -typedef ze_result_t(ZE_APICALL *zeMemGetPitchFor2dImage_pfn)( - ze_context_handle_t hContext, ze_device_handle_t hDevice, size_t imageWidth, - size_t imageHeight, unsigned int elementSizeInBytes, size_t *rowPitch); +#include "loader/ze_loader.h" namespace { -zeMemGetPitchFor2dImage_pfn zeMemGetPitchFor2dImageFunctionPtr = nullptr; - -zeImageGetDeviceOffsetExp_pfn zeImageGetDeviceOffsetExpFunctionPtr = nullptr; - -// TODO: unify with image_helpers.cpp/ur2zeImageDesc -static ur_result_t -ur2zeImageDescBindless(const ur_image_format_t *ImageFormat, - const ur_image_desc_t *ImageDesc, - ZeStruct &ZeImageDesc) { - auto [ZeImageFormatType, ZeImageFormatTypeSize] = - getImageFormatTypeAndSize(ImageFormat); - if (ZeImageFormatTypeSize == 0) { - return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; - } - // TODO: populate the layout mapping - ze_image_format_layout_t ZeImageFormatLayout; - switch (ImageFormat->channelOrder) { - case UR_IMAGE_CHANNEL_ORDER_A: - case UR_IMAGE_CHANNEL_ORDER_R: { - switch (ZeImageFormatTypeSize) { - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32; - break; - default: - logger::error("ur2zeImageDescBindless: unexpected data type size"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - } - case UR_IMAGE_CHANNEL_ORDER_RG: - case UR_IMAGE_CHANNEL_ORDER_RA: - case UR_IMAGE_CHANNEL_ORDER_RX: { - switch (ZeImageFormatTypeSize) { - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32; - break; - default: - logger::error("ur2zeImageDescBindless: unexpected data type size"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - } - case UR_IMAGE_CHANNEL_ORDER_RGBX: - case UR_IMAGE_CHANNEL_ORDER_RGBA: - case UR_IMAGE_CHANNEL_ORDER_ARGB: - case UR_IMAGE_CHANNEL_ORDER_BGRA: { - switch (ZeImageFormatTypeSize) { - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; - break; - default: - logger::error("ur2zeImageDescBindless: unexpected data type size"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - } +bool Is3ChannelOrder(ur_image_channel_order_t ChannelOrder) { + switch (ChannelOrder) { case UR_IMAGE_CHANNEL_ORDER_RGB: - case UR_IMAGE_CHANNEL_ORDER_RGX: { - switch (ZeImageFormatTypeSize) { - case 8: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8; - break; - case 16: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16; - break; - case 32: - ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32; - break; - default: - logger::error("ur2zeImageDescBindless: unexpected data type size"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - break; - } - default: - logger::error("format channel order = {}", ImageFormat->channelOrder); - die("ur2zeImageDescBindless: unsupported image channel order\n"); - break; - } - - ze_image_format_t ZeFormatDesc = { - ZeImageFormatLayout, ZeImageFormatType, - // TODO: are swizzles deducted from image_format->image_channel_order? - ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, - ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_A}; - - ze_image_type_t ZeImageType; - switch (ImageDesc->type) { - case UR_MEM_TYPE_IMAGE1D: - ZeImageType = ZE_IMAGE_TYPE_1D; - break; - case UR_MEM_TYPE_IMAGE2D: - ZeImageType = ZE_IMAGE_TYPE_2D; - break; - case UR_MEM_TYPE_IMAGE3D: - ZeImageType = ZE_IMAGE_TYPE_3D; - break; - case UR_MEM_TYPE_IMAGE1D_ARRAY: - ZeImageType = ZE_IMAGE_TYPE_1DARRAY; - break; - case UR_MEM_TYPE_IMAGE2D_ARRAY: - ZeImageType = ZE_IMAGE_TYPE_2DARRAY; - break; + case UR_IMAGE_CHANNEL_ORDER_RGX: + return true; default: - logger::error("ur2zeImageDescBindless: unsupported image type"); - return UR_RESULT_ERROR_INVALID_VALUE; - } - - ZeImageDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; - ZeImageDesc.pNext = ImageDesc->pNext; - ZeImageDesc.type = ZeImageType; - ZeImageDesc.format = ZeFormatDesc; - ZeImageDesc.width = ur_cast(ImageDesc->width); - ZeImageDesc.height = - std::max(ur_cast(ImageDesc->height), (uint64_t)1); - ZeImageDesc.depth = - std::max(ur_cast(ImageDesc->depth), (uint64_t)1); - ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); - ZeImageDesc.miplevels = ImageDesc->numMipLevel; - - return UR_RESULT_SUCCESS; -} - -ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, - const ur_image_format_t *pImageFormat, - const ur_image_desc_t *pImageDesc, - ur_sampler_handle_t hSampler, - ur_exp_image_native_handle_t *phImage) { - std::shared_lock Lock(hContext->Mutex); - - UR_ASSERT(hContext && hDevice && hImageMem, - UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(pImageFormat && pImageDesc && phImage, - UR_RESULT_ERROR_INVALID_NULL_POINTER); - - ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDescBindless(pImageFormat, pImageDesc, ZeImageDesc)); - - ZeStruct BindlessDesc; - BindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; - ZeImageDesc.pNext = &BindlessDesc; - - ZeStruct ZeSamplerDesc; - if (hSampler) { - ZeSamplerDesc = hSampler->ZeSamplerDesc; - BindlessDesc.pNext = &ZeSamplerDesc; - BindlessDesc.flags |= ZE_IMAGE_BINDLESS_EXP_FLAG_SAMPLED_IMAGE; - } - - ze_image_handle_t ZeImage; - - ze_memory_allocation_properties_t MemAllocProperties{ - ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES, nullptr, - ZE_MEMORY_TYPE_UNKNOWN, 0, 0}; - ZE2UR_CALL(zeMemGetAllocProperties, - (hContext->ZeContext, reinterpret_cast(hImageMem), - &MemAllocProperties, nullptr)); - if (MemAllocProperties.type == ZE_MEMORY_TYPE_UNKNOWN) { - ur_image *UrImage = reinterpret_cast(hImageMem); - - ZE2UR_CALL(zeImageViewCreateExt, - (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, - UrImage->ZeImage, &ZeImage)); - ZE2UR_CALL(zeContextMakeImageResident, - (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); - } else if (MemAllocProperties.type == ZE_MEMORY_TYPE_DEVICE || - MemAllocProperties.type == ZE_MEMORY_TYPE_HOST || - MemAllocProperties.type == ZE_MEMORY_TYPE_SHARED) { - ZeStruct PitchedDesc; - PitchedDesc.ptr = reinterpret_cast(hImageMem); - if (hSampler) { - ZeSamplerDesc.pNext = &PitchedDesc; - } else { - BindlessDesc.pNext = &PitchedDesc; - } - - ZE2UR_CALL(zeImageCreate, (hContext->ZeContext, hDevice->ZeDevice, - &ZeImageDesc, &ZeImage)); - ZE2UR_CALL(zeContextMakeImageResident, - (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); - } else { - return UR_RESULT_ERROR_INVALID_VALUE; + return false; } - - static std::once_flag InitFlag; - std::call_once(InitFlag, [&]() { - ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; - auto Result = zeDriverGetExtensionFunctionAddress( - DriverHandle, "zeImageGetDeviceOffsetExp", - (void **)&zeImageGetDeviceOffsetExpFunctionPtr); - if (Result != ZE_RESULT_SUCCESS) - logger::error("zeDriverGetExtensionFunctionAddress " - "zeImageGetDeviceOffsetExpv failed, err = {}", - Result); - }); - if (!zeImageGetDeviceOffsetExpFunctionPtr) - return UR_RESULT_ERROR_INVALID_OPERATION; - - uint64_t DeviceOffset{}; - ze_image_handle_t ZeImageTranslated; - ZE2UR_CALL(zelLoaderTranslateHandle, - (ZEL_HANDLE_IMAGE, ZeImage, (void **)&ZeImageTranslated)); - ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr, - (ZeImageTranslated, &DeviceOffset)); - *phImage = DeviceOffset; - - hDevice->ZeOffsetToImageHandleMap[*phImage] = ZeImage; - - return UR_RESULT_SUCCESS; } } // namespace namespace ur::level_zero { -ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t widthInBytes, - size_t height, size_t elementSizeBytes, - void **ppMem, size_t *pResultPitch) { - std::shared_lock Lock(hContext->Mutex); - - UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(widthInBytes != 0, UR_RESULT_ERROR_INVALID_USM_SIZE); - UR_ASSERT(ppMem && pResultPitch, UR_RESULT_ERROR_INVALID_NULL_POINTER); - - static std::once_flag InitFlag; - std::call_once(InitFlag, [&]() { - ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; - auto Result = zeDriverGetExtensionFunctionAddress( - DriverHandle, "zeMemGetPitchFor2dImage", - (void **)&zeMemGetPitchFor2dImageFunctionPtr); - if (Result != ZE_RESULT_SUCCESS) - logger::error( - "zeDriverGetExtensionFunctionAddress zeMemGetPitchFor2dImage " - "failed, err = {}", - Result); - }); - if (!zeMemGetPitchFor2dImageFunctionPtr) - return UR_RESULT_ERROR_INVALID_OPERATION; - - size_t Width = widthInBytes / elementSizeBytes; - size_t RowPitch; - ze_device_handle_t ZeDeviceTranslated; - ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_DEVICE, hDevice->ZeDevice, - (void **)&ZeDeviceTranslated)); - ZE2UR_CALL(zeMemGetPitchFor2dImageFunctionPtr, - (hContext->ZeContext, ZeDeviceTranslated, Width, height, - elementSizeBytes, &RowPitch)); - *pResultPitch = RowPitch; - - size_t Size = height * RowPitch; - UR_CALL(ur::level_zero::urUSMDeviceAlloc(hContext, hDevice, pUSMDesc, pool, - Size, ppMem)); - - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_native_handle_t hImage) { - UR_ASSERT(hContext && hDevice && hImage, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - - auto item = hDevice->ZeOffsetToImageHandleMap.find(hImage); - - if (item != hDevice->ZeOffsetToImageHandleMap.end()) { - if (checkL0LoaderTeardown()) { - ZE2UR_CALL(zeImageDestroy, (item->second)); - } - hDevice->ZeOffsetToImageHandleMap.erase(item); - } else { - return UR_RESULT_ERROR_INVALID_NULL_HANDLE; - } - - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesSampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_native_handle_t hImage) { - // Sampled image is a combination of unsampled image and sampler. - // Sampler is released in urSamplerRelease. - return ur::level_zero::urBindlessImagesUnsampledImageHandleDestroyExp( - hContext, hDevice, hImage); -} - -ur_result_t urBindlessImagesImageAllocateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_image_mem_native_handle_t *phImageMem) { - std::shared_lock Lock(hContext->Mutex); - - UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(pImageFormat && pImageDesc && phImageMem, - UR_RESULT_ERROR_INVALID_NULL_POINTER); - - ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDescBindless(pImageFormat, pImageDesc, ZeImageDesc)); - - ze_image_bindless_exp_desc_t ZeImageBindlessDesc; - ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; - ZeImageBindlessDesc.pNext = nullptr; - ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; - ZeImageDesc.pNext = &ZeImageBindlessDesc; - - ze_image_handle_t ZeImage; - ZE2UR_CALL(zeImageCreate, - (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, &ZeImage)); - ZE2UR_CALL(zeContextMakeImageResident, - (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); - UR_CALL(createUrMemFromZeImage(hContext, ZeImage, /*OwnZeMemHandle*/ true, - ZeImageDesc, phImageMem)); - return UR_RESULT_SUCCESS; -} - ur_result_t urBindlessImagesImageFreeExp(ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, @@ -371,38 +44,14 @@ urBindlessImagesImageFreeExp(ur_context_handle_t /*hContext*/, return UR_RESULT_SUCCESS; } -ur_result_t urBindlessImagesUnsampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_image_native_handle_t *phImage) { - UR_CALL(bindlessImagesCreateImpl(hContext, hDevice, hImageMem, pImageFormat, - pImageDesc, nullptr, phImage)); - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesSampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_sampler_handle_t hSampler, ur_exp_image_native_handle_t *phImage) { - UR_CALL(bindlessImagesCreateImpl(hContext, hDevice, hImageMem, pImageFormat, - pImageDesc, hSampler, phImage)); - return UR_RESULT_SUCCESS; -} - ur_result_t urBindlessImagesImageCopyExp( - ur_queue_handle_t hQueue, [[maybe_unused]] const void *pSrc, - [[maybe_unused]] void *pDst, - [[maybe_unused]] const ur_image_desc_t *pSrcImageDesc, - [[maybe_unused]] const ur_image_desc_t *pDstImageDesc, - [[maybe_unused]] const ur_image_format_t *pSrcImageFormat, - [[maybe_unused]] const ur_image_format_t *pDstImageFormat, - [[maybe_unused]] ur_exp_image_copy_region_t *pCopyRegion, - [[maybe_unused]] ur_exp_image_copy_flags_t imageCopyFlags, - [[maybe_unused]] uint32_t numEventsInWaitList, - [[maybe_unused]] const ur_event_handle_t *phEventWaitList, - [[maybe_unused]] ur_event_handle_t *phEvent) { + ur_queue_handle_t hQueue, const void *pSrc, void *pDst, + const ur_image_desc_t *pSrcImageDesc, const ur_image_desc_t *pDstImageDesc, + const ur_image_format_t *pSrcImageFormat, + const ur_image_format_t *pDstImageFormat, + ur_exp_image_copy_region_t *pCopyRegion, + ur_exp_image_copy_flags_t imageCopyFlags, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { std::scoped_lock Lock(hQueue->Mutex); UR_ASSERT(hQueue, UR_RESULT_ERROR_INVALID_NULL_HANDLE); @@ -416,9 +65,6 @@ ur_result_t urBindlessImagesImageCopyExp( UR_ASSERT(!(pSrcImageDesc && UR_MEM_TYPE_IMAGE1D_ARRAY < pSrcImageDesc->type), UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR); - ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDescBindless(pSrcImageFormat, pSrcImageDesc, ZeImageDesc)); - bool UseCopyEngine = hQueue->useCopyEngine(/*PreferCopyEngine*/ true); // Due to the limitation of the copy engine, disable usage of Copy Engine // Given 3 channel image @@ -458,493 +104,15 @@ ur_result_t urBindlessImagesImageCopyExp( const auto &ZeCommandList = CommandList->first; const auto &WaitList = (*Event)->WaitList; - if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_HOST_TO_DEVICE) { - uint32_t SrcRowPitch = - pSrcImageDesc->width * getPixelSizeBytes(pSrcImageFormat); - uint32_t SrcSlicePitch = SrcRowPitch * pSrcImageDesc->height; - if (pDstImageDesc->rowPitch == 0) { - // Copy to Non-USM memory - ze_image_region_t DstRegion; - UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset, - &pCopyRegion->copyExtent, DstRegion)); - auto *UrImage = static_cast(pDst); - const char *SrcPtr = - static_cast(pSrc) + - pCopyRegion->srcOffset.z * SrcSlicePitch + - pCopyRegion->srcOffset.y * SrcRowPitch + - pCopyRegion->srcOffset.x * getPixelSizeBytes(pSrcImageFormat); - ZE2UR_CALL(zeCommandListAppendImageCopyFromMemoryExt, - (ZeCommandList, UrImage->ZeImage, SrcPtr, &DstRegion, - SrcRowPitch, SrcSlicePitch, ZeEvent, WaitList.Length, - WaitList.ZeEventList)); - } else { - // Copy to pitched USM memory - uint32_t DstRowPitch = pDstImageDesc->rowPitch; - ze_copy_region_t ZeDstRegion = {(uint32_t)pCopyRegion->dstOffset.x, - (uint32_t)pCopyRegion->dstOffset.y, - (uint32_t)pCopyRegion->dstOffset.z, - DstRowPitch, - (uint32_t)pCopyRegion->copyExtent.height, - (uint32_t)pCopyRegion->copyExtent.depth}; - uint32_t DstSlicePitch = 0; - ze_copy_region_t ZeSrcRegion = {(uint32_t)pCopyRegion->srcOffset.x, - (uint32_t)pCopyRegion->srcOffset.y, - (uint32_t)pCopyRegion->srcOffset.z, - SrcRowPitch, - (uint32_t)pCopyRegion->copyExtent.height, - (uint32_t)pCopyRegion->copyExtent.depth}; - ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, - (ZeCommandList, pDst, &ZeDstRegion, DstRowPitch, DstSlicePitch, - pSrc, &ZeSrcRegion, SrcRowPitch, SrcSlicePitch, ZeEvent, - WaitList.Length, WaitList.ZeEventList)); - } - } else if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_HOST) { - uint32_t DstRowPitch = - pDstImageDesc->width * getPixelSizeBytes(pDstImageFormat); - uint32_t DstSlicePitch = DstRowPitch * pDstImageDesc->height; - if (pSrcImageDesc->rowPitch == 0) { - // Copy from Non-USM memory to host - ze_image_region_t SrcRegion; - UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset, - &pCopyRegion->copyExtent, SrcRegion)); - auto *UrImage = static_cast(pSrc); - char *DstPtr = - static_cast(pDst) + pCopyRegion->dstOffset.z * DstSlicePitch + - pCopyRegion->dstOffset.y * DstRowPitch + - pCopyRegion->dstOffset.x * getPixelSizeBytes(pDstImageFormat); - ZE2UR_CALL(zeCommandListAppendImageCopyToMemoryExt, - (ZeCommandList, DstPtr, UrImage->ZeImage, &SrcRegion, - DstRowPitch, DstSlicePitch, ZeEvent, WaitList.Length, - WaitList.ZeEventList)); - } else { - // Copy from pitched USM memory to host - ze_copy_region_t ZeDstRegion = {(uint32_t)pCopyRegion->dstOffset.x, - (uint32_t)pCopyRegion->dstOffset.y, - (uint32_t)pCopyRegion->dstOffset.z, - DstRowPitch, - (uint32_t)pCopyRegion->copyExtent.height, - (uint32_t)pCopyRegion->copyExtent.depth}; - uint32_t SrcRowPitch = pSrcImageDesc->rowPitch; - ze_copy_region_t ZeSrcRegion = {(uint32_t)pCopyRegion->srcOffset.x, - (uint32_t)pCopyRegion->srcOffset.y, - (uint32_t)pCopyRegion->srcOffset.z, - SrcRowPitch, - (uint32_t)pCopyRegion->copyExtent.height, - (uint32_t)pCopyRegion->copyExtent.depth}; - uint32_t SrcSlicePitch = 0; - ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, - (ZeCommandList, pDst, &ZeDstRegion, DstRowPitch, DstSlicePitch, - pSrc, &ZeSrcRegion, SrcRowPitch, SrcSlicePitch, ZeEvent, - WaitList.Length, WaitList.ZeEventList)); - } - } else if (imageCopyFlags == UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_DEVICE) { - ze_image_region_t DstRegion; - UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->dstOffset, - &pCopyRegion->copyExtent, DstRegion)); - ze_image_region_t SrcRegion; - UR_CALL(getImageRegionHelper(ZeImageDesc, &pCopyRegion->srcOffset, - &pCopyRegion->copyExtent, SrcRegion)); - auto *UrImageDst = static_cast(pDst); - auto *UrImageSrc = static_cast(pSrc); - ZE2UR_CALL(zeCommandListAppendImageCopyRegion, - (ZeCommandList, UrImageDst->ZeImage, UrImageSrc->ZeImage, - &DstRegion, &SrcRegion, ZeEvent, WaitList.Length, - WaitList.ZeEventList)); - } else { - logger::error("urBindlessImagesImageCopyExp: unexpected imageCopyFlags"); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - - UR_CALL(hQueue->executeCommandList(CommandList, Blocking, OkToBatch)); - - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesImageGetInfoExp( - ur_context_handle_t, ur_exp_image_mem_native_handle_t hImageMem, - ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { - UR_ASSERT(hImageMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(UR_IMAGE_INFO_DEPTH >= propName, - UR_RESULT_ERROR_INVALID_ENUMERATION); - UR_ASSERT(pPropValue || pPropSizeRet, UR_RESULT_ERROR_INVALID_NULL_POINTER); - - auto *UrImage = reinterpret_cast(hImageMem); - ze_image_desc_t &Desc = UrImage->ZeImageDesc; - switch (propName) { - case UR_IMAGE_INFO_WIDTH: - if (pPropValue) { - *(uint64_t *)pPropValue = Desc.width; - } - if (pPropSizeRet) { - *pPropSizeRet = sizeof(uint64_t); - } - return UR_RESULT_SUCCESS; - case UR_IMAGE_INFO_HEIGHT: - if (pPropValue) { - *(uint32_t *)pPropValue = Desc.height; - } - if (pPropSizeRet) { - *pPropSizeRet = sizeof(uint32_t); - } - return UR_RESULT_SUCCESS; - case UR_IMAGE_INFO_DEPTH: - if (pPropValue) { - *(uint32_t *)pPropValue = Desc.depth; - } - if (pPropSizeRet) { - *pPropSizeRet = sizeof(uint32_t); - } - return UR_RESULT_SUCCESS; - case UR_IMAGE_INFO_FORMAT: - if (pPropValue) { - ur_image_format_t UrImageFormat; - UR_CALL(ze2urImageFormat(&Desc, &UrImageFormat)); - *(ur_image_format_t *)pPropValue = UrImageFormat; - } - if (pPropSizeRet) { - *pPropSizeRet = sizeof(ur_image_format_t); - } - return UR_RESULT_SUCCESS; - default: - return UR_RESULT_ERROR_INVALID_VALUE; - } -} - -ur_result_t urBindlessImagesMipmapGetLevelExp( - ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, - ur_exp_image_mem_native_handle_t /*hImageMem*/, uint32_t /*mipmapLevel*/, - ur_exp_image_mem_native_handle_t * /*phImageMem*/) { - logger::error(logger::LegacyMessage("[UR][L0] {} function not implemented!"), - "{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t -urBindlessImagesMipmapFreeExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hMem) { - return ur::level_zero::urBindlessImagesImageFreeExp(hContext, hDevice, hMem); -} - -ur_result_t urBindlessImagesImportExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, - ur_exp_external_mem_type_t memHandleType, - ur_exp_external_mem_desc_t *pExternalMemDesc, - ur_exp_external_mem_handle_t *phExternalMem) { - - UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(pExternalMemDesc && phExternalMem, - UR_RESULT_ERROR_INVALID_NULL_POINTER); - - struct ur_ze_external_memory_data *externalMemoryData = - new struct ur_ze_external_memory_data; - - void *pNext = const_cast(pExternalMemDesc->pNext); - while (pNext != nullptr) { - const ur_base_desc_t *BaseDesc = static_cast(pNext); - if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_FILE_DESCRIPTOR) { - ze_external_memory_import_fd_t *importFd = - new ze_external_memory_import_fd_t; - importFd->stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_FD; - importFd->pNext = nullptr; - auto FileDescriptor = - static_cast(pNext); - importFd->fd = FileDescriptor->fd; - importFd->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; - externalMemoryData->importExtensionDesc = importFd; - externalMemoryData->type = UR_ZE_EXTERNAL_OPAQUE_FD; - } else if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_WIN32_HANDLE) { - ze_external_memory_import_win32_handle_t *importWin32 = - new ze_external_memory_import_win32_handle_t; - importWin32->stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32; - importWin32->pNext = nullptr; - auto Win32Handle = static_cast(pNext); - - switch (memHandleType) { - case UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT: - importWin32->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_WIN32; - break; - case UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX12_RESOURCE: - importWin32->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_D3D12_RESOURCE; - break; - case UR_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD: - default: - delete importWin32; - delete externalMemoryData; - return UR_RESULT_ERROR_INVALID_VALUE; - } - importWin32->handle = Win32Handle->handle; - externalMemoryData->importExtensionDesc = importWin32; - externalMemoryData->type = UR_ZE_EXTERNAL_WIN32; - } - pNext = const_cast(BaseDesc->pNext); - } - externalMemoryData->size = size; - - *phExternalMem = - reinterpret_cast(externalMemoryData); - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesMapExternalArrayExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_external_mem_handle_t hExternalMem, - ur_exp_image_mem_native_handle_t *phImageMem) { - - UR_ASSERT(hContext && hDevice && hExternalMem, - UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(pImageFormat && pImageDesc, UR_RESULT_ERROR_INVALID_NULL_POINTER); - - struct ur_ze_external_memory_data *externalMemoryData = - reinterpret_cast(hExternalMem); - - ze_image_bindless_exp_desc_t ZeImageBindlessDesc = {}; - ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; - - ZeStruct ZeImageDesc; - UR_CALL(ur2zeImageDescBindless(pImageFormat, pImageDesc, ZeImageDesc)); - - ZeImageBindlessDesc.pNext = externalMemoryData->importExtensionDesc; - ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; - ZeImageDesc.pNext = &ZeImageBindlessDesc; - - ze_image_handle_t ZeImage; - ZE2UR_CALL(zeImageCreate, - (hContext->ZeContext, hDevice->ZeDevice, &ZeImageDesc, &ZeImage)); - ZE2UR_CALL(zeContextMakeImageResident, - (hContext->ZeContext, hDevice->ZeDevice, ZeImage)); - UR_CALL(createUrMemFromZeImage(hContext, ZeImage, /*OwnZeMemHandle*/ true, - ZeImageDesc, phImageMem)); - externalMemoryData->urMemoryHandle = - reinterpret_cast(*phImageMem); - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesMapExternalLinearMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, uint64_t offset, - uint64_t size, ur_exp_external_mem_handle_t hExternalMem, void **phRetMem) { - UR_ASSERT(hContext && hDevice && hExternalMem, - UR_RESULT_ERROR_INVALID_NULL_HANDLE); - UR_ASSERT(offset && size, UR_RESULT_ERROR_INVALID_BUFFER_SIZE); - - struct ur_ze_external_memory_data *externalMemoryData = - reinterpret_cast(hExternalMem); - UR_ASSERT(externalMemoryData && externalMemoryData->importExtensionDesc, - UR_RESULT_ERROR_INVALID_NULL_POINTER); - - ze_device_mem_alloc_desc_t allocDesc = {}; - allocDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; - allocDesc.flags = 0; - allocDesc.pNext = externalMemoryData->importExtensionDesc; - void *mappedMemory; - - ze_result_t zeResult = zeMemAllocDevice(hContext->ZeContext, &allocDesc, size, - 1, hDevice->ZeDevice, &mappedMemory); - if (zeResult != ZE_RESULT_SUCCESS) { - return UR_RESULT_ERROR_OUT_OF_RESOURCES; - } - - zeResult = zeContextMakeMemoryResident(hContext->ZeContext, hDevice->ZeDevice, - mappedMemory, size); - if (zeResult != ZE_RESULT_SUCCESS) { - zeMemFree(hContext->ZeContext, mappedMemory); - return UR_RESULT_ERROR_UNKNOWN; - } - *phRetMem = reinterpret_cast( - reinterpret_cast(mappedMemory) + offset); - - externalMemoryData->urMemoryHandle = - reinterpret_cast(*phRetMem); + auto res = bindlessImagesHandleCopyFlags( + pSrc, pDst, pSrcImageDesc, pDstImageDesc, pSrcImageFormat, + pDstImageFormat, pCopyRegion, imageCopyFlags, ZeCommandList, ZeEvent, + WaitList.Length, WaitList.ZeEventList); - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesReleaseExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_mem_handle_t hExternalMem) { - - UR_ASSERT(hContext && hDevice && hExternalMem, - UR_RESULT_ERROR_INVALID_NULL_HANDLE); - - struct ur_ze_external_memory_data *externalMemoryData = - reinterpret_cast(hExternalMem); - - UR_CALL(ur::level_zero::urMemRelease(externalMemoryData->urMemoryHandle)); - - switch (externalMemoryData->type) { - case UR_ZE_EXTERNAL_OPAQUE_FD: - delete (reinterpret_cast( - externalMemoryData->importExtensionDesc)); - break; - case UR_ZE_EXTERNAL_WIN32: - delete (reinterpret_cast( - externalMemoryData->importExtensionDesc)); - break; - default: - return UR_RESULT_ERROR_INVALID_VALUE; - } - - delete (externalMemoryData); - - return UR_RESULT_SUCCESS; -} - -ur_result_t urBindlessImagesImportExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_semaphore_type_t semHandleType, - ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, - ur_exp_external_semaphore_handle_t *phExternalSemaphoreHandle) { - - auto UrPlatform = hContext->getPlatform(); - if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { - logger::error(logger::LegacyMessage("[UR][L0] "), - " {} function not supported!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - if (UrPlatform->ZeExternalSemaphoreExt.LoaderExtension) { - ze_external_semaphore_ext_desc_t SemDesc = { - ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_EXT_DESC, nullptr, - ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_OPAQUE_FD}; - ze_external_semaphore_ext_handle_t ExtSemaphoreHandle; - ze_external_semaphore_fd_ext_desc_t FDExpDesc = { - ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_FD_EXT_DESC, nullptr, 0}; - ze_external_semaphore_win32_ext_desc_t Win32ExpDesc = { - ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_WIN32_EXT_DESC, nullptr, nullptr, - nullptr}; - void *pNext = const_cast(pExternalSemaphoreDesc->pNext); - while (pNext != nullptr) { - const ur_base_desc_t *BaseDesc = - static_cast(pNext); - if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_FILE_DESCRIPTOR) { - auto FileDescriptor = - static_cast(pNext); - FDExpDesc.fd = FileDescriptor->fd; - SemDesc.pNext = &FDExpDesc; - switch (semHandleType) { - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_OPAQUE_FD: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_OPAQUE_FD; - break; - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_FD: - SemDesc.flags = - ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_VK_TIMELINE_SEMAPHORE_FD; - break; - default: - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_WIN32_HANDLE) { - SemDesc.pNext = &Win32ExpDesc; - auto Win32Handle = static_cast(pNext); - switch (semHandleType) { - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_OPAQUE_WIN32; - break; - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT_DX12_FENCE: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_D3D12_FENCE; - break; - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_WIN32_NT: - SemDesc.flags = - ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_VK_TIMELINE_SEMAPHORE_WIN32; - break; - default: - return UR_RESULT_ERROR_INVALID_VALUE; - } - Win32ExpDesc.handle = Win32Handle->handle; - } - pNext = const_cast(BaseDesc->pNext); - } - ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt.zexImportExternalSemaphoreExp, - (hDevice->ZeDevice, &SemDesc, &ExtSemaphoreHandle)); - *phExternalSemaphoreHandle = - (ur_exp_external_semaphore_handle_t)ExtSemaphoreHandle; - - } else { - ze_intel_external_semaphore_exp_desc_t SemDesc = { - ZE_INTEL_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_EXP_DESC, nullptr, - ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_OPAQUE_FD}; - ze_intel_external_semaphore_exp_handle_t ExtSemaphoreHandle; - ze_intel_external_semaphore_desc_fd_exp_desc_t FDExpDesc = { - ZE_INTEL_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_FD_EXP_DESC, nullptr, 0}; - _ze_intel_external_semaphore_win32_exp_desc_t Win32ExpDesc = { - ZE_INTEL_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_WIN32_EXP_DESC, nullptr, - nullptr, nullptr}; - void *pNext = const_cast(pExternalSemaphoreDesc->pNext); - while (pNext != nullptr) { - const ur_base_desc_t *BaseDesc = - static_cast(pNext); - if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_FILE_DESCRIPTOR) { - auto FileDescriptor = - static_cast(pNext); - FDExpDesc.fd = FileDescriptor->fd; - SemDesc.pNext = &FDExpDesc; - switch (semHandleType) { - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_OPAQUE_FD: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_OPAQUE_FD; - break; - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_FD: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_TIMELINE_SEMAPHORE_FD; - break; - default: - return UR_RESULT_ERROR_INVALID_VALUE; - } - } else if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_WIN32_HANDLE) { - SemDesc.pNext = &Win32ExpDesc; - auto Win32Handle = static_cast(pNext); - switch (semHandleType) { - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_OPAQUE_WIN32; - break; - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT_DX12_FENCE: - SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_D3D12_FENCE; - break; - case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_WIN32_NT: - SemDesc.flags = - ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_TIMELINE_SEMAPHORE_WIN32; - break; - default: - return UR_RESULT_ERROR_INVALID_VALUE; - } - Win32ExpDesc.handle = Win32Handle->handle; - } - pNext = const_cast(BaseDesc->pNext); - } - - ze_device_handle_t translatedDevice; - ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_DEVICE, hDevice->ZeDevice, - (void **)&translatedDevice)); - // If the L0 loader is not aware of the extension, the handles need to be - // translated - ZE2UR_CALL( - UrPlatform->ZeExternalSemaphoreExt.zexExpImportExternalSemaphoreExp, - (translatedDevice, &SemDesc, &ExtSemaphoreHandle)); - - *phExternalSemaphoreHandle = - (ur_exp_external_semaphore_handle_t)ExtSemaphoreHandle; - } - - return UR_RESULT_SUCCESS; -} + if (res == UR_RESULT_SUCCESS) + UR_CALL(hQueue->executeCommandList(CommandList, Blocking, OkToBatch)); -ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t /*hDevice*/, - ur_exp_external_semaphore_handle_t hExternalSemaphore) { - auto UrPlatform = hContext->getPlatform(); - if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { - logger::error(logger::LegacyMessage("[UR][L0] "), - " {} function not supported!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; - } - if (UrPlatform->ZeExternalSemaphoreExt.LoaderExtension) { - ZE2UR_CALL( - UrPlatform->ZeExternalSemaphoreExt.zexDeviceReleaseExternalSemaphoreExp, - ((ze_external_semaphore_ext_handle_t)hExternalSemaphore)); - } else { - ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt - .zexExpDeviceReleaseExternalSemaphoreExp, - ((ze_intel_external_semaphore_exp_handle_t)hExternalSemaphore)); - } - - return UR_RESULT_SUCCESS; + return res; } ur_result_t urBindlessImagesWaitExternalSemaphoreExp( diff --git a/unified-runtime/source/adapters/level_zero/image_common.cpp b/unified-runtime/source/adapters/level_zero/image_common.cpp new file mode 100644 index 0000000000000..d80ca7d9b8522 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/image_common.cpp @@ -0,0 +1,1399 @@ +//===--------- image.cpp - Level Zero Adapter -----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include + +#include "common.hpp" +#ifdef UR_ADAPTER_LEVEL_ZERO_V2 +#include "v2/context.hpp" +#else +#include "context.hpp" +#endif +#include "helpers/memory_helpers.hpp" +#include "image_common.hpp" +#include "logger/ur_logger.hpp" +#include "sampler.hpp" +#include "ur_interface_loader.hpp" + +typedef ze_result_t(ZE_APICALL *zeMemGetPitchFor2dImage_pfn)( + ze_context_handle_t hContext, ze_device_handle_t hDevice, size_t imageWidth, + size_t imageHeight, unsigned int elementSizeInBytes, size_t *rowPitch); + +typedef ze_result_t(ZE_APICALL *zeImageGetDeviceOffsetExp_pfn)( + ze_image_handle_t hImage, uint64_t *pDeviceOffset); + +zeMemGetPitchFor2dImage_pfn zeMemGetPitchFor2dImageFunctionPtr = nullptr; +zeImageGetDeviceOffsetExp_pfn zeImageGetDeviceOffsetExpFunctionPtr = nullptr; + +namespace { + +/// Construct UR image format from ZE image desc. +ur_result_t ze2urImageFormat(const ze_image_format_t &ZeImageFormat, + ur_image_format_t *UrImageFormat) { + size_t ZeImageFormatTypeSize; + switch (ZeImageFormat.layout) { + case ZE_IMAGE_FORMAT_LAYOUT_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: + ZeImageFormatTypeSize = 8; + break; + case ZE_IMAGE_FORMAT_LAYOUT_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: + ZeImageFormatTypeSize = 16; + break; + case ZE_IMAGE_FORMAT_LAYOUT_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: + ZeImageFormatTypeSize = 32; + break; + default: + logger::error( + "ze2urImageFormat: unsupported image format layout: layout = {}", + ZeImageFormat.layout); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ur_image_channel_order_t ChannelOrder; + switch (ZeImageFormat.layout) { + case ZE_IMAGE_FORMAT_LAYOUT_8: + case ZE_IMAGE_FORMAT_LAYOUT_16: + case ZE_IMAGE_FORMAT_LAYOUT_32: + switch (ZeImageFormat.x) { + case ZE_IMAGE_FORMAT_SWIZZLE_R: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_R; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_A; + break; + default: + logger::error( + "ze2urImageFormat: unexpected image format channel x: x = {}", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32: + if (ZeImageFormat.x != ZE_IMAGE_FORMAT_SWIZZLE_R) { + logger::error( + "ze2urImageFormat: unexpected image format channel x: x = {}", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + switch (ZeImageFormat.y) { + case ZE_IMAGE_FORMAT_SWIZZLE_G: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RG; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RA; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RX; + break; + default: + logger::error( + "ze2urImageFormat: unexpected image format channel y: y = {}\n", + ZeImageFormat.y); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32: + if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G) { + switch (ZeImageFormat.z) { + case ZE_IMAGE_FORMAT_SWIZZLE_B: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGB; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGX; + break; + default: + logger::error( + "ze2urImageFormat: unexpected image format channel z: z = {}\n", + ZeImageFormat.z); + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else { + logger::error("ze2urImageFormat: unexpected image format channel"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8: + case ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16: + case ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32: + if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_B) { + switch (ZeImageFormat.w) { + case ZE_IMAGE_FORMAT_SWIZZLE_X: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBX; + break; + case ZE_IMAGE_FORMAT_SWIZZLE_A: + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_RGBA; + break; + default: + logger::error("ze2urImageFormat: unexpected image format channel w: w " + "= {}", + ZeImageFormat.x); + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_A && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_B) { + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_ARGB; + } else if (ZeImageFormat.x == ZE_IMAGE_FORMAT_SWIZZLE_B && + ZeImageFormat.y == ZE_IMAGE_FORMAT_SWIZZLE_G && + ZeImageFormat.z == ZE_IMAGE_FORMAT_SWIZZLE_R && + ZeImageFormat.w == ZE_IMAGE_FORMAT_SWIZZLE_A) { + ChannelOrder = UR_IMAGE_CHANNEL_ORDER_BGRA; + } else { + logger::error("ze2urImageFormat: unexpected image format channel"); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + default: + logger::error( + "ze2urImageFormat: unsupported image format layout: layout = {}", + ZeImageFormat.layout); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + ur_image_channel_type_t ChannelType; + switch (ZeImageFormat.type) { + case ZE_IMAGE_FORMAT_TYPE_UINT: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_SINT: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_UNORM: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_UNORM_INT16; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_SNORM: + switch (ZeImageFormatTypeSize) { + case 8: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT8; + break; + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_SNORM_INT16; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + case ZE_IMAGE_FORMAT_TYPE_FLOAT: + switch (ZeImageFormatTypeSize) { + case 16: + ChannelType = UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT; + break; + case 32: + ChannelType = UR_IMAGE_CHANNEL_TYPE_FLOAT; + break; + default: + logger::error("ze2urImageFormat: unexpected image format type size: size " + "= {}", + ZeImageFormatTypeSize); + return UR_RESULT_ERROR_INVALID_VALUE; + } + break; + default: + logger::error("ze2urImageFormat: unsupported image format type: type = {}", + ZeImageFormat.type); + return UR_RESULT_ERROR_INVALID_VALUE; + } + + UrImageFormat->channelOrder = ChannelOrder; + UrImageFormat->channelType = ChannelType; + return UR_RESULT_SUCCESS; +} + +/// Construct UR bindless image struct from ZE image handle and desc. +ur_result_t createUrImgFromZeImage(ze_context_handle_t hContext, + ze_device_handle_t hDevice, + const ZeStruct &ZeImageDesc, + ur_exp_image_mem_native_handle_t *pImg) { + ze_image_handle_t ZeImage; + ZE2UR_CALL(zeImageCreate, (hContext, hDevice, &ZeImageDesc, &ZeImage)); + ZE2UR_CALL(zeContextMakeImageResident, (hContext, hDevice, ZeImage)); + + try { + ur_bindless_mem_handle_t *urImg = + new ur_bindless_mem_handle_t(ZeImage, ZeImageDesc); + *pImg = reinterpret_cast(urImg); + } catch (const std::bad_alloc &) { + return UR_RESULT_ERROR_OUT_OF_HOST_MEMORY; + } catch (...) { + return UR_RESULT_ERROR_UNKNOWN; + } + return UR_RESULT_SUCCESS; +} + +ur_result_t bindlessImagesCreateImpl(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, + const ur_image_desc_t *pImageDesc, + ur_sampler_handle_t hSampler, + ur_exp_image_native_handle_t *phImage) { + UR_ASSERT(hContext && hDevice && hImageMem, + UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pImageFormat && pImageDesc && phImage, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + + ZeStruct ZeImageDesc; + UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + + ZeStruct BindlessDesc; + BindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; + ZeImageDesc.pNext = &BindlessDesc; + + ZeStruct ZeSamplerDesc; + if (hSampler) { + ZeSamplerDesc = hSampler->ZeSamplerDesc; + BindlessDesc.pNext = &ZeSamplerDesc; + BindlessDesc.flags |= ZE_IMAGE_BINDLESS_EXP_FLAG_SAMPLED_IMAGE; + } + + ze_image_handle_t ZeImage; + + ze_memory_allocation_properties_t MemAllocProperties{ + ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES, nullptr, + ZE_MEMORY_TYPE_UNKNOWN, 0, 0}; + + ze_context_handle_t zeCtx = hContext->getZeHandle(); + + ZE2UR_CALL(zeMemGetAllocProperties, + (zeCtx, reinterpret_cast(hImageMem), + &MemAllocProperties, nullptr)); + if (MemAllocProperties.type == ZE_MEMORY_TYPE_UNKNOWN) { + ur_bindless_mem_handle_t *urImg = + reinterpret_cast(hImageMem); + ze_image_handle_t zeImg1 = urImg->getZeImage(); + + ZE2UR_CALL(zeImageViewCreateExt, + (zeCtx, hDevice->ZeDevice, &ZeImageDesc, zeImg1, &ZeImage)); + ZE2UR_CALL(zeContextMakeImageResident, (zeCtx, hDevice->ZeDevice, ZeImage)); + } else if (MemAllocProperties.type == ZE_MEMORY_TYPE_DEVICE || + MemAllocProperties.type == ZE_MEMORY_TYPE_HOST || + MemAllocProperties.type == ZE_MEMORY_TYPE_SHARED) { + ZeStruct PitchedDesc; + PitchedDesc.ptr = reinterpret_cast(hImageMem); + if (hSampler) { + ZeSamplerDesc.pNext = &PitchedDesc; + } else { + BindlessDesc.pNext = &PitchedDesc; + } + + ZE2UR_CALL(zeImageCreate, + (zeCtx, hDevice->ZeDevice, &ZeImageDesc, &ZeImage)); + ZE2UR_CALL(zeContextMakeImageResident, (zeCtx, hDevice->ZeDevice, ZeImage)); + } else { + return UR_RESULT_ERROR_INVALID_VALUE; + } + + static std::once_flag InitFlag; + std::call_once(InitFlag, [&]() { + ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; + auto Result = zeDriverGetExtensionFunctionAddress( + DriverHandle, "zeImageGetDeviceOffsetExp", + (void **)&zeImageGetDeviceOffsetExpFunctionPtr); + if (Result != ZE_RESULT_SUCCESS) + logger::error("zeDriverGetExtensionFunctionAddress " + "zeImageGetDeviceOffsetExpv failed, err = {}", + Result); + }); + if (!zeImageGetDeviceOffsetExpFunctionPtr) + return UR_RESULT_ERROR_INVALID_OPERATION; + uint64_t DeviceOffset{}; + ze_image_handle_t ZeImageTranslated; + ZE2UR_CALL(zelLoaderTranslateHandle, + (ZEL_HANDLE_IMAGE, ZeImage, (void **)&ZeImageTranslated)); + ZE2UR_CALL(zeImageGetDeviceOffsetExpFunctionPtr, + (ZeImageTranslated, &DeviceOffset)); + *phImage = DeviceOffset; + + std::shared_lock Lock(hDevice->Mutex); + hDevice->ZeOffsetToImageHandleMap[*phImage] = ZeImage; + Lock.release(); + return UR_RESULT_SUCCESS; +} + +/// Return element size in bytes of a pixel. +uint32_t getPixelSizeBytes(const ur_image_format_t *Format) { + uint32_t NumChannels = 0; + switch (Format->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_A: + case UR_IMAGE_CHANNEL_ORDER_R: + case UR_IMAGE_CHANNEL_ORDER_INTENSITY: + case UR_IMAGE_CHANNEL_ORDER_LUMINANCE: + case UR_IMAGE_CHANNEL_ORDER_FORCE_UINT32: + NumChannels = 1; + break; + case UR_IMAGE_CHANNEL_ORDER_RG: + case UR_IMAGE_CHANNEL_ORDER_RA: + case UR_IMAGE_CHANNEL_ORDER_RX: + NumChannels = 2; + break; + case UR_IMAGE_CHANNEL_ORDER_RGB: + case UR_IMAGE_CHANNEL_ORDER_RGX: + NumChannels = 3; + break; + case UR_IMAGE_CHANNEL_ORDER_RGBA: + case UR_IMAGE_CHANNEL_ORDER_BGRA: + case UR_IMAGE_CHANNEL_ORDER_ARGB: + case UR_IMAGE_CHANNEL_ORDER_ABGR: + case UR_IMAGE_CHANNEL_ORDER_RGBX: + case UR_IMAGE_CHANNEL_ORDER_SRGBA: + NumChannels = 4; + break; + default: + ur::unreachable(); + } + uint32_t ChannelTypeSizeInBytes = 0; + switch (Format->channelType) { + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: + ChannelTypeSizeInBytes = 1; + break; + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: + case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_565: + case UR_IMAGE_CHANNEL_TYPE_UNORM_SHORT_555: + case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: + ChannelTypeSizeInBytes = 2; + break; + case UR_IMAGE_CHANNEL_TYPE_INT_101010: + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: + case UR_IMAGE_CHANNEL_TYPE_FLOAT: + case UR_IMAGE_CHANNEL_TYPE_FORCE_UINT32: + ChannelTypeSizeInBytes = 4; + break; + default: + ur::unreachable(); + } + return NumChannels * ChannelTypeSizeInBytes; +} + +std::pair +getImageFormatTypeAndSize(const ur_image_format_t *ImageFormat) { + ze_image_format_type_t ZeImageFormatType; + size_t ZeImageFormatTypeSize; + switch (ImageFormat->channelType) { + case UR_IMAGE_CHANNEL_TYPE_FLOAT: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_HALF_FLOAT: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FLOAT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT32: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNSIGNED_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UINT; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_UNORM_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_UNORM; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT32: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 32; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SIGNED_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SINT; + ZeImageFormatTypeSize = 8; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT16: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; + ZeImageFormatTypeSize = 16; + break; + } + case UR_IMAGE_CHANNEL_TYPE_SNORM_INT8: { + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_SNORM; + ZeImageFormatTypeSize = 8; + break; + } + default: + logger::error("ur2zeImageDesc: unsupported image data type: data type = {}", + ImageFormat->channelType); + ZeImageFormatType = ZE_IMAGE_FORMAT_TYPE_FORCE_UINT32; + ZeImageFormatTypeSize = 0; + } + return {ZeImageFormatType, ZeImageFormatTypeSize}; +} + +} // namespace + +/// Construct ZE image desc from UR image format and desc. +ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, + const ur_image_desc_t *ImageDesc, + ZeStruct &ZeImageDesc) { + + auto [ZeImageFormatType, ZeImageFormatTypeSize] = + getImageFormatTypeAndSize(ImageFormat); + + if (ZeImageFormatTypeSize == 0) { + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + + // TODO: populate the layout mapping + ze_image_format_layout_t ZeImageFormatLayout; + switch (ImageFormat->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_R: + case UR_IMAGE_CHANNEL_ORDER_A: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32; + break; + default: + logger::error("ur2zeImageDesc: unexpected data type Size\n"); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + break; + } + case UR_IMAGE_CHANNEL_ORDER_RG: + case UR_IMAGE_CHANNEL_ORDER_RA: + case UR_IMAGE_CHANNEL_ORDER_RX: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32; + break; + default: + logger::error("ur2zeImageDesc: unexpected data type Size\n"); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + break; + } + case UR_IMAGE_CHANNEL_ORDER_RGB: + case UR_IMAGE_CHANNEL_ORDER_RGX: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32; + break; + default: + logger::error("ur2zeImageDesc: unexpected data type size"); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + break; + } + case UR_IMAGE_CHANNEL_ORDER_RGBA: + case UR_IMAGE_CHANNEL_ORDER_RGBX: + case UR_IMAGE_CHANNEL_ORDER_BGRA: + case UR_IMAGE_CHANNEL_ORDER_ARGB: + case UR_IMAGE_CHANNEL_ORDER_ABGR: { + switch (ZeImageFormatTypeSize) { + case 8: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_8_8_8_8; + break; + case 16: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_16_16_16_16; + break; + case 32: + ZeImageFormatLayout = ZE_IMAGE_FORMAT_LAYOUT_32_32_32_32; + break; + default: + logger::error("ur2zeImageDesc: unexpected data type Size\n"); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + break; + } + default: + logger::error("format layout = {}", ImageFormat->channelOrder); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + break; + } + + ze_image_format_t ZeFormatDesc; + switch (ImageFormat->channelOrder) { + case UR_IMAGE_CHANNEL_ORDER_R: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_0, + ZE_IMAGE_FORMAT_SWIZZLE_0, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_A: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_A, ZE_IMAGE_FORMAT_SWIZZLE_0, + ZE_IMAGE_FORMAT_SWIZZLE_0, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_RG: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_0, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_RA: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_A, + ZE_IMAGE_FORMAT_SWIZZLE_0, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_RX: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_X, + ZE_IMAGE_FORMAT_SWIZZLE_0, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_RGB: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_RGX: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_X, ZE_IMAGE_FORMAT_SWIZZLE_1}; + break; + case UR_IMAGE_CHANNEL_ORDER_RGBA: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_A}; + break; + case UR_IMAGE_CHANNEL_ORDER_RGBX: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_X}; + break; + case UR_IMAGE_CHANNEL_ORDER_BGRA: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_B, ZE_IMAGE_FORMAT_SWIZZLE_G, + ZE_IMAGE_FORMAT_SWIZZLE_R, ZE_IMAGE_FORMAT_SWIZZLE_A}; + break; + case UR_IMAGE_CHANNEL_ORDER_ARGB: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_A, ZE_IMAGE_FORMAT_SWIZZLE_R, + ZE_IMAGE_FORMAT_SWIZZLE_G, ZE_IMAGE_FORMAT_SWIZZLE_B}; + break; + case UR_IMAGE_CHANNEL_ORDER_ABGR: + ZeFormatDesc = {ZeImageFormatLayout, ZeImageFormatType, + ZE_IMAGE_FORMAT_SWIZZLE_A, ZE_IMAGE_FORMAT_SWIZZLE_B, + ZE_IMAGE_FORMAT_SWIZZLE_G, ZE_IMAGE_FORMAT_SWIZZLE_R}; + break; + default: + logger::error("ur2zeImageDesc: unsupported image channel order"); + return UR_RESULT_ERROR_UNSUPPORTED_IMAGE_FORMAT; + } + ze_image_type_t ZeImageType; + switch (ImageDesc->type) { + case UR_MEM_TYPE_IMAGE1D: + ZeImageType = ZE_IMAGE_TYPE_1D; + break; + case UR_MEM_TYPE_IMAGE2D: + ZeImageType = ZE_IMAGE_TYPE_2D; + break; + case UR_MEM_TYPE_IMAGE3D: + ZeImageType = ZE_IMAGE_TYPE_3D; + break; + case UR_MEM_TYPE_IMAGE1D_ARRAY: + ZeImageType = ZE_IMAGE_TYPE_1DARRAY; + break; + case UR_MEM_TYPE_IMAGE2D_ARRAY: + ZeImageType = ZE_IMAGE_TYPE_2DARRAY; + break; + default: + logger::error("ur2zeImageDesc: unsupported image type"); + return UR_RESULT_ERROR_INVALID_IMAGE_FORMAT_DESCRIPTOR; + } + ZeImageDesc.stype = ZE_STRUCTURE_TYPE_IMAGE_DESC; + ZeImageDesc.pNext = ImageDesc->pNext; + ZeImageDesc.flags = 0; + ZeImageDesc.type = ZeImageType; + ZeImageDesc.format = ZeFormatDesc; + ZeImageDesc.width = ur_cast(ImageDesc->width); + ZeImageDesc.height = + std::max(ur_cast(ImageDesc->height), (uint64_t)1); + ZeImageDesc.depth = + std::max(ur_cast(ImageDesc->depth), (uint64_t)1); + + ZeImageDesc.arraylevels = ur_cast(ImageDesc->arraySize); + ZeImageDesc.miplevels = ImageDesc->numMipLevel; + return UR_RESULT_SUCCESS; +} + +ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, + ur_rect_offset_t *Origin, + ur_rect_region_t *Region, + ze_image_region_t &ZeRegion) { + UR_ASSERT(Origin, UR_RESULT_ERROR_INVALID_VALUE); + UR_ASSERT(Region, UR_RESULT_ERROR_INVALID_VALUE); + + if (ZeImageDesc.type == ZE_IMAGE_TYPE_1D || + ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) { + Region->height = 1; + Region->depth = 1; + } else if (ZeImageDesc.type == ZE_IMAGE_TYPE_2D || + ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) { + Region->depth = 1; + } + +#ifndef NDEBUG + UR_ASSERT((ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Origin->y == 0 && + Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Origin->z == 0) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), + UR_RESULT_ERROR_INVALID_VALUE); + + UR_ASSERT(Region->width && Region->height && Region->depth, + UR_RESULT_ERROR_INVALID_VALUE); + UR_ASSERT( + (ZeImageDesc.type == ZE_IMAGE_TYPE_1D && Region->height == 1 && + Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY && Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2D && Region->depth == 1) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) || + (ZeImageDesc.type == ZE_IMAGE_TYPE_3D), + UR_RESULT_ERROR_INVALID_VALUE); +#endif // !NDEBUG + + uint32_t OriginX = ur_cast(Origin->x); + uint32_t OriginY = ur_cast(Origin->y); + uint32_t OriginZ = ur_cast(Origin->z); + + uint32_t Width = ur_cast(Region->width); + uint32_t Height = (ZeImageDesc.type == ZE_IMAGE_TYPE_1DARRAY) + ? ZeImageDesc.arraylevels + : ur_cast(Region->height); + uint32_t Depth = (ZeImageDesc.type == ZE_IMAGE_TYPE_2DARRAY) + ? ZeImageDesc.arraylevels + : ur_cast(Region->depth); + + ZeRegion = {OriginX, OriginY, OriginZ, Width, Height, Depth}; + + return UR_RESULT_SUCCESS; +} + +ur_result_t bindlessImagesHandleCopyFlags( + const void *pSrc, void *pDst, const ur_image_desc_t *pSrcImageDesc, + const ur_image_desc_t *pDstImageDesc, + const ur_image_format_t *pSrcImageFormat, + const ur_image_format_t *pDstImageFormat, + ur_exp_image_copy_region_t *pCopyRegion, + ur_exp_image_copy_flags_t imageCopyFlags, + ze_command_list_handle_t ZeCommandList, ze_event_handle_t zeSignalEvent, + uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents) { + + ZeStruct zeSrcImageDesc; + ur2zeImageDesc(pSrcImageFormat, pSrcImageDesc, zeSrcImageDesc); + + switch (imageCopyFlags) { + case UR_EXP_IMAGE_COPY_FLAG_HOST_TO_DEVICE: { + uint32_t SrcRowPitch = + pSrcImageDesc->width * getPixelSizeBytes(pSrcImageFormat); + uint32_t SrcSlicePitch = SrcRowPitch * pSrcImageDesc->height; + if (pDstImageDesc->rowPitch == 0) { + // Copy to Non-USM memory + + ze_image_region_t DstRegion; + UR_CALL(getImageRegionHelper(zeSrcImageDesc, &pCopyRegion->dstOffset, + &pCopyRegion->copyExtent, DstRegion)); + auto *urDstImg = static_cast(pDst); + + const char *SrcPtr = + static_cast(pSrc) + + pCopyRegion->srcOffset.z * SrcSlicePitch + + pCopyRegion->srcOffset.y * SrcRowPitch + + pCopyRegion->srcOffset.x * getPixelSizeBytes(pSrcImageFormat); + + ZE2UR_CALL(zeCommandListAppendImageCopyFromMemoryExt, + (ZeCommandList, urDstImg->getZeImage(), SrcPtr, &DstRegion, + SrcRowPitch, SrcSlicePitch, zeSignalEvent, numWaitEvents, + phWaitEvents)); + } else { + // Copy to pitched USM memory + uint32_t DstRowPitch = pDstImageDesc->rowPitch; + ze_copy_region_t ZeDstRegion = {(uint32_t)pCopyRegion->dstOffset.x, + (uint32_t)pCopyRegion->dstOffset.y, + (uint32_t)pCopyRegion->dstOffset.z, + DstRowPitch, + (uint32_t)pCopyRegion->copyExtent.height, + (uint32_t)pCopyRegion->copyExtent.depth}; + uint32_t DstSlicePitch = 0; + ze_copy_region_t ZeSrcRegion = {(uint32_t)pCopyRegion->srcOffset.x, + (uint32_t)pCopyRegion->srcOffset.y, + (uint32_t)pCopyRegion->srcOffset.z, + SrcRowPitch, + (uint32_t)pCopyRegion->copyExtent.height, + (uint32_t)pCopyRegion->copyExtent.depth}; + ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, + (ZeCommandList, pDst, &ZeDstRegion, DstRowPitch, DstSlicePitch, + pSrc, &ZeSrcRegion, SrcRowPitch, SrcSlicePitch, zeSignalEvent, + numWaitEvents, phWaitEvents)); + } + return UR_RESULT_SUCCESS; + }; + case UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_HOST: { + uint32_t DstRowPitch = + pDstImageDesc->width * getPixelSizeBytes(pDstImageFormat); + uint32_t DstSlicePitch = DstRowPitch * pDstImageDesc->height; + if (pSrcImageDesc->rowPitch == 0) { + // Copy from Non-USM memory to host + ze_image_region_t SrcRegion; + UR_CALL(getImageRegionHelper(zeSrcImageDesc, &pCopyRegion->srcOffset, + &pCopyRegion->copyExtent, SrcRegion)); + + auto *urSrcImg = reinterpret_cast(pSrc); + + char *DstPtr = + static_cast(pDst) + pCopyRegion->dstOffset.z * DstSlicePitch + + pCopyRegion->dstOffset.y * DstRowPitch + + pCopyRegion->dstOffset.x * getPixelSizeBytes(pDstImageFormat); + ZE2UR_CALL(zeCommandListAppendImageCopyToMemoryExt, + (ZeCommandList, DstPtr, urSrcImg->getZeImage(), &SrcRegion, + DstRowPitch, DstSlicePitch, zeSignalEvent, numWaitEvents, + phWaitEvents)); + } else { + // Copy from pitched USM memory to host + ze_copy_region_t ZeDstRegion = {(uint32_t)pCopyRegion->dstOffset.x, + (uint32_t)pCopyRegion->dstOffset.y, + (uint32_t)pCopyRegion->dstOffset.z, + DstRowPitch, + (uint32_t)pCopyRegion->copyExtent.height, + (uint32_t)pCopyRegion->copyExtent.depth}; + uint32_t SrcRowPitch = pSrcImageDesc->rowPitch; + ze_copy_region_t ZeSrcRegion = {(uint32_t)pCopyRegion->srcOffset.x, + (uint32_t)pCopyRegion->srcOffset.y, + (uint32_t)pCopyRegion->srcOffset.z, + SrcRowPitch, + (uint32_t)pCopyRegion->copyExtent.height, + (uint32_t)pCopyRegion->copyExtent.depth}; + uint32_t SrcSlicePitch = 0; + ZE2UR_CALL(zeCommandListAppendMemoryCopyRegion, + (ZeCommandList, pDst, &ZeDstRegion, DstRowPitch, DstSlicePitch, + pSrc, &ZeSrcRegion, SrcRowPitch, SrcSlicePitch, zeSignalEvent, + numWaitEvents, phWaitEvents)); + } + return UR_RESULT_SUCCESS; + }; + case UR_EXP_IMAGE_COPY_FLAG_DEVICE_TO_DEVICE: { + ze_image_region_t DstRegion; + UR_CALL(getImageRegionHelper(zeSrcImageDesc, &pCopyRegion->dstOffset, + &pCopyRegion->copyExtent, DstRegion)); + ze_image_region_t SrcRegion; + UR_CALL(getImageRegionHelper(zeSrcImageDesc, &pCopyRegion->srcOffset, + &pCopyRegion->copyExtent, SrcRegion)); + + auto *urImgSrc = reinterpret_cast(pSrc); + auto *urImgDst = reinterpret_cast(pDst); + + ZE2UR_CALL(zeCommandListAppendImageCopyRegion, + (ZeCommandList, urImgDst->getZeImage(), urImgSrc->getZeImage(), + &DstRegion, &SrcRegion, zeSignalEvent, numWaitEvents, + phWaitEvents)); + + return UR_RESULT_SUCCESS; + }; + default: + logger::error("ur_queue_immediate_in_order_t::bindlessImagesImageCopyExp: " + "unexpected imageCopyFlags"); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } +} + +namespace ur::level_zero { + +ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + const ur_usm_desc_t *pUSMDesc, + ur_usm_pool_handle_t pool, size_t widthInBytes, + size_t height, size_t elementSizeBytes, + void **ppMem, size_t *pResultPitch) { + UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(widthInBytes != 0, UR_RESULT_ERROR_INVALID_USM_SIZE); + UR_ASSERT(ppMem && pResultPitch, UR_RESULT_ERROR_INVALID_NULL_POINTER); + + static std::once_flag InitFlag; + std::call_once(InitFlag, [&]() { + ze_driver_handle_t DriverHandle = hContext->getPlatform()->ZeDriver; + auto Result = zeDriverGetExtensionFunctionAddress( + DriverHandle, "zeMemGetPitchFor2dImage", + (void **)&zeMemGetPitchFor2dImageFunctionPtr); + if (Result != ZE_RESULT_SUCCESS) + logger::error( + "zeDriverGetExtensionFunctionAddress zeMemGetPitchFor2dImage " + "failed, err = {}", + Result); + }); + if (!zeMemGetPitchFor2dImageFunctionPtr) + return UR_RESULT_ERROR_INVALID_OPERATION; + + size_t Width = widthInBytes / elementSizeBytes; + size_t RowPitch; + ze_device_handle_t ZeDeviceTranslated; + ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_DEVICE, hDevice->ZeDevice, + (void **)&ZeDeviceTranslated)); + ZE2UR_CALL(zeMemGetPitchFor2dImageFunctionPtr, + (hContext->getZeHandle(), ZeDeviceTranslated, Width, height, + elementSizeBytes, &RowPitch)); + *pResultPitch = RowPitch; + + size_t Size = height * RowPitch; + UR_CALL(ur::level_zero::urUSMDeviceAlloc(hContext, hDevice, pUSMDesc, pool, + Size, ppMem)); + + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesImageAllocateExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_mem_native_handle_t *phImageMem) { + UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pImageFormat && pImageDesc && phImageMem, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + + ZeStruct ZeImageDesc; + UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + + ze_image_bindless_exp_desc_t ZeImageBindlessDesc; + ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; + ZeImageBindlessDesc.pNext = nullptr; + ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; + ZeImageDesc.pNext = &ZeImageBindlessDesc; + + UR_CALL(createUrImgFromZeImage(hContext->getZeHandle(), hDevice->ZeDevice, + ZeImageDesc, phImageMem)); + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesUnsampledImageCreateExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_image_native_handle_t *phImage) { + UR_CALL(bindlessImagesCreateImpl(hContext, hDevice, hImageMem, pImageFormat, + pImageDesc, nullptr, phImage)); + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesSampledImageCreateExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_sampler_handle_t hSampler, ur_exp_image_native_handle_t *phImage) { + UR_CALL(bindlessImagesCreateImpl(hContext, hDevice, hImageMem, pImageFormat, + pImageDesc, hSampler, phImage)); + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage) { + UR_ASSERT(hContext && hDevice && hImage, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + + std::shared_lock Lock(hDevice->Mutex); + auto item = hDevice->ZeOffsetToImageHandleMap.find(hImage); + + if (item != hDevice->ZeOffsetToImageHandleMap.end()) { + hDevice->ZeOffsetToImageHandleMap.erase(item); + Lock.release(); + ZE2UR_CALL(zeImageDestroy, (item->second)); + } else { + Lock.release(); + return UR_RESULT_ERROR_INVALID_NULL_HANDLE; + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesSampledImageHandleDestroyExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_image_native_handle_t hImage) { + // Sampled image is a combination of unsampled image and sampler. + // Sampler is released in urSamplerRelease. + return ur::level_zero::urBindlessImagesUnsampledImageHandleDestroyExp( + hContext, hDevice, hImage); +} + +ur_result_t +urBindlessImagesMipmapFreeExp(ur_context_handle_t hContext, + ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hMem) { + return ur::level_zero::urBindlessImagesImageFreeExp(hContext, hDevice, hMem); +} + +ur_result_t urBindlessImagesImageGetInfoExp( + ur_context_handle_t, ur_exp_image_mem_native_handle_t hImageMem, + ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { + UR_ASSERT(hImageMem, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(UR_IMAGE_INFO_DEPTH >= propName, + UR_RESULT_ERROR_INVALID_ENUMERATION); + UR_ASSERT(pPropValue || pPropSizeRet, UR_RESULT_ERROR_INVALID_NULL_POINTER); + + ur_bindless_mem_handle_t *urImg = + reinterpret_cast(hImageMem); + + switch (propName) { + case UR_IMAGE_INFO_WIDTH: + if (pPropValue) { + *(uint64_t *)pPropValue = urImg->getWidth(); /* Desc.width; */ + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(uint64_t); + } + return UR_RESULT_SUCCESS; + case UR_IMAGE_INFO_HEIGHT: + if (pPropValue) { + *(uint32_t *)pPropValue = urImg->getHeight(); /* Desc.height; */ + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(uint32_t); + } + return UR_RESULT_SUCCESS; + case UR_IMAGE_INFO_DEPTH: + if (pPropValue) { + *(uint32_t *)pPropValue = urImg->getDepth(); /* Desc.depth; */ + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(uint32_t); + } + return UR_RESULT_SUCCESS; + case UR_IMAGE_INFO_FORMAT: + if (pPropValue) { + ur_image_format_t UrImageFormat; + UR_CALL(ze2urImageFormat(urImg->getFormat(), &UrImageFormat)); + *(ur_image_format_t *)pPropValue = UrImageFormat; + } + if (pPropSizeRet) { + *pPropSizeRet = sizeof(ur_image_format_t); + } + return UR_RESULT_SUCCESS; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } +} + +ur_result_t urBindlessImagesMipmapGetLevelExp( + ur_context_handle_t /*hContext*/, ur_device_handle_t /*hDevice*/, + ur_exp_image_mem_native_handle_t /*hImageMem*/, uint32_t /*mipmapLevel*/, + ur_exp_image_mem_native_handle_t * /*phImageMem*/) { + logger::error(logger::LegacyMessage("[UR][L0] {} function not implemented!"), + "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; +} + +ur_result_t urBindlessImagesImportExternalMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, + ur_exp_external_mem_type_t memHandleType, + ur_exp_external_mem_desc_t *pExternalMemDesc, + ur_exp_external_mem_handle_t *phExternalMem) { + + UR_ASSERT(hContext && hDevice, UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pExternalMemDesc && phExternalMem, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + + struct ur_ze_external_memory_data *externalMemoryData = + new struct ur_ze_external_memory_data; + + void *pNext = const_cast(pExternalMemDesc->pNext); + while (pNext != nullptr) { + const ur_base_desc_t *BaseDesc = static_cast(pNext); + if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_FILE_DESCRIPTOR) { + ze_external_memory_import_fd_t *importFd = + new ze_external_memory_import_fd_t; + importFd->stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_FD; + importFd->pNext = nullptr; + auto FileDescriptor = + static_cast(pNext); + importFd->fd = FileDescriptor->fd; + importFd->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_FD; + externalMemoryData->importExtensionDesc = importFd; + externalMemoryData->type = UR_ZE_EXTERNAL_OPAQUE_FD; + } else if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_WIN32_HANDLE) { + ze_external_memory_import_win32_handle_t *importWin32 = + new ze_external_memory_import_win32_handle_t; + importWin32->stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_IMPORT_WIN32; + importWin32->pNext = nullptr; + auto Win32Handle = static_cast(pNext); + + switch (memHandleType) { + case UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT: + importWin32->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_OPAQUE_WIN32; + break; + case UR_EXP_EXTERNAL_MEM_TYPE_WIN32_NT_DX12_RESOURCE: + importWin32->flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_D3D12_RESOURCE; + break; + case UR_EXP_EXTERNAL_MEM_TYPE_OPAQUE_FD: + default: + delete importWin32; + delete externalMemoryData; + return UR_RESULT_ERROR_INVALID_VALUE; + } + importWin32->handle = Win32Handle->handle; + externalMemoryData->importExtensionDesc = importWin32; + externalMemoryData->type = UR_ZE_EXTERNAL_WIN32; + } + pNext = const_cast(BaseDesc->pNext); + } + externalMemoryData->size = size; + + *phExternalMem = + reinterpret_cast(externalMemoryData); + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesMapExternalArrayExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, + ur_exp_external_mem_handle_t hExternalMem, + ur_exp_image_mem_native_handle_t *phImageMem) { + + UR_ASSERT(hContext && hDevice && hExternalMem, + UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(pImageFormat && pImageDesc, UR_RESULT_ERROR_INVALID_NULL_POINTER); + + struct ur_ze_external_memory_data *externalMemoryData = + reinterpret_cast(hExternalMem); + + ze_image_bindless_exp_desc_t ZeImageBindlessDesc = {}; + ZeImageBindlessDesc.stype = ZE_STRUCTURE_TYPE_BINDLESS_IMAGE_EXP_DESC; + + ZeStruct ZeImageDesc; + UR_CALL(ur2zeImageDesc(pImageFormat, pImageDesc, ZeImageDesc)); + + ZeImageBindlessDesc.pNext = externalMemoryData->importExtensionDesc; + ZeImageBindlessDesc.flags = ZE_IMAGE_BINDLESS_EXP_FLAG_BINDLESS; + ZeImageDesc.pNext = &ZeImageBindlessDesc; + + UR_CALL(createUrImgFromZeImage(hContext->getZeHandle(), hDevice->ZeDevice, + ZeImageDesc, phImageMem)); + + externalMemoryData->urMemoryHandle = + reinterpret_cast(*phImageMem); + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesReleaseExternalMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_mem_handle_t hExternalMem) { + + UR_ASSERT(hContext && hDevice && hExternalMem, + UR_RESULT_ERROR_INVALID_NULL_HANDLE); + + struct ur_ze_external_memory_data *externalMemoryData = + reinterpret_cast(hExternalMem); + + UR_CALL(ur::level_zero::urMemRelease(externalMemoryData->urMemoryHandle)); + + switch (externalMemoryData->type) { + case UR_ZE_EXTERNAL_OPAQUE_FD: + delete (reinterpret_cast( + externalMemoryData->importExtensionDesc)); + break; + case UR_ZE_EXTERNAL_WIN32: + delete (reinterpret_cast( + externalMemoryData->importExtensionDesc)); + break; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } + + delete (externalMemoryData); + + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesImportExternalSemaphoreExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, + ur_exp_external_semaphore_type_t semHandleType, + ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, + ur_exp_external_semaphore_handle_t *phExternalSemaphoreHandle) { + + auto UrPlatform = hContext->getPlatform(); + if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { + logger::error(logger::LegacyMessage("[UR][L0] "), + " {} function not supported!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + if (UrPlatform->ZeExternalSemaphoreExt.LoaderExtension) { + ze_external_semaphore_ext_desc_t SemDesc = { + ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_EXT_DESC, nullptr, + ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_OPAQUE_FD}; + ze_external_semaphore_ext_handle_t ExtSemaphoreHandle; + ze_external_semaphore_fd_ext_desc_t FDExpDesc = { + ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_FD_EXT_DESC, nullptr, 0}; + ze_external_semaphore_win32_ext_desc_t Win32ExpDesc = { + ZE_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_WIN32_EXT_DESC, nullptr, nullptr, + nullptr}; + void *pNext = const_cast(pExternalSemaphoreDesc->pNext); + while (pNext != nullptr) { + const ur_base_desc_t *BaseDesc = + static_cast(pNext); + if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_FILE_DESCRIPTOR) { + auto FileDescriptor = + static_cast(pNext); + FDExpDesc.fd = FileDescriptor->fd; + SemDesc.pNext = &FDExpDesc; + switch (semHandleType) { + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_OPAQUE_FD: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_OPAQUE_FD; + break; + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_FD: + SemDesc.flags = + ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_VK_TIMELINE_SEMAPHORE_FD; + break; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_WIN32_HANDLE) { + SemDesc.pNext = &Win32ExpDesc; + auto Win32Handle = static_cast(pNext); + switch (semHandleType) { + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_OPAQUE_WIN32; + break; + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT_DX12_FENCE: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_D3D12_FENCE; + break; + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_WIN32_NT: + SemDesc.flags = + ZE_EXTERNAL_SEMAPHORE_EXT_FLAG_VK_TIMELINE_SEMAPHORE_WIN32; + break; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } + Win32ExpDesc.handle = Win32Handle->handle; + } + pNext = const_cast(BaseDesc->pNext); + } + ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt.zexImportExternalSemaphoreExp, + (hDevice->ZeDevice, &SemDesc, &ExtSemaphoreHandle)); + *phExternalSemaphoreHandle = + (ur_exp_external_semaphore_handle_t)ExtSemaphoreHandle; + + } else { + ze_intel_external_semaphore_exp_desc_t SemDesc = { + ZE_INTEL_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_EXP_DESC, nullptr, + ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_OPAQUE_FD}; + ze_intel_external_semaphore_exp_handle_t ExtSemaphoreHandle; + ze_intel_external_semaphore_desc_fd_exp_desc_t FDExpDesc = { + ZE_INTEL_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_FD_EXP_DESC, nullptr, 0}; + _ze_intel_external_semaphore_win32_exp_desc_t Win32ExpDesc = { + ZE_INTEL_STRUCTURE_TYPE_EXTERNAL_SEMAPHORE_WIN32_EXP_DESC, nullptr, + nullptr, nullptr}; + void *pNext = const_cast(pExternalSemaphoreDesc->pNext); + while (pNext != nullptr) { + const ur_base_desc_t *BaseDesc = + static_cast(pNext); + if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_FILE_DESCRIPTOR) { + auto FileDescriptor = + static_cast(pNext); + FDExpDesc.fd = FileDescriptor->fd; + SemDesc.pNext = &FDExpDesc; + switch (semHandleType) { + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_OPAQUE_FD: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_OPAQUE_FD; + break; + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_FD: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_TIMELINE_SEMAPHORE_FD; + break; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } + } else if (BaseDesc->stype == UR_STRUCTURE_TYPE_EXP_WIN32_HANDLE) { + SemDesc.pNext = &Win32ExpDesc; + auto Win32Handle = static_cast(pNext); + switch (semHandleType) { + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_OPAQUE_WIN32; + break; + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_WIN32_NT_DX12_FENCE: + SemDesc.flags = ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_D3D12_FENCE; + break; + case UR_EXP_EXTERNAL_SEMAPHORE_TYPE_TIMELINE_WIN32_NT: + SemDesc.flags = + ZE_EXTERNAL_SEMAPHORE_EXP_FLAGS_TIMELINE_SEMAPHORE_WIN32; + break; + default: + return UR_RESULT_ERROR_INVALID_VALUE; + } + Win32ExpDesc.handle = Win32Handle->handle; + } + pNext = const_cast(BaseDesc->pNext); + } + + ze_device_handle_t translatedDevice; + ZE2UR_CALL(zelLoaderTranslateHandle, (ZEL_HANDLE_DEVICE, hDevice->ZeDevice, + (void **)&translatedDevice)); + // If the L0 loader is not aware of the extension, the handles need to be + // translated + ZE2UR_CALL( + UrPlatform->ZeExternalSemaphoreExt.zexExpImportExternalSemaphoreExp, + (translatedDevice, &SemDesc, &ExtSemaphoreHandle)); + + *phExternalSemaphoreHandle = + (ur_exp_external_semaphore_handle_t)ExtSemaphoreHandle; + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( + ur_context_handle_t hContext, [[maybe_unused]] ur_device_handle_t hDevice, + ur_exp_external_semaphore_handle_t hExternalSemaphore) { + auto UrPlatform = hContext->getPlatform(); + if (UrPlatform->ZeExternalSemaphoreExt.Supported == false) { + logger::error(logger::LegacyMessage("[UR][L0] "), + " {} function not supported!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + } + if (UrPlatform->ZeExternalSemaphoreExt.LoaderExtension) { + ZE2UR_CALL( + UrPlatform->ZeExternalSemaphoreExt.zexDeviceReleaseExternalSemaphoreExp, + ((ze_external_semaphore_ext_handle_t)hExternalSemaphore)); + } else { + ZE2UR_CALL(UrPlatform->ZeExternalSemaphoreExt + .zexExpDeviceReleaseExternalSemaphoreExp, + ((ze_intel_external_semaphore_exp_handle_t)hExternalSemaphore)); + } + + return UR_RESULT_SUCCESS; +} + +ur_result_t urBindlessImagesMapExternalLinearMemoryExp( + ur_context_handle_t hContext, ur_device_handle_t hDevice, uint64_t offset, + uint64_t size, ur_exp_external_mem_handle_t hExternalMem, void **phRetMem) { + UR_ASSERT(hContext && hDevice && hExternalMem, + UR_RESULT_ERROR_INVALID_NULL_HANDLE); + UR_ASSERT(offset && size, UR_RESULT_ERROR_INVALID_BUFFER_SIZE); + + struct ur_ze_external_memory_data *externalMemoryData = + reinterpret_cast(hExternalMem); + UR_ASSERT(externalMemoryData && externalMemoryData->importExtensionDesc, + UR_RESULT_ERROR_INVALID_NULL_POINTER); + + ze_device_mem_alloc_desc_t allocDesc = {}; + allocDesc.stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC; + allocDesc.flags = 0; + allocDesc.pNext = externalMemoryData->importExtensionDesc; + void *mappedMemory; + + ze_result_t zeResult = + zeMemAllocDevice(hContext->getZeHandle(), &allocDesc, size, 1, + hDevice->ZeDevice, &mappedMemory); + if (zeResult != ZE_RESULT_SUCCESS) { + return UR_RESULT_ERROR_OUT_OF_RESOURCES; + } + + zeResult = zeContextMakeMemoryResident(hContext->getZeHandle(), + hDevice->ZeDevice, mappedMemory, size); + if (zeResult != ZE_RESULT_SUCCESS) { + zeMemFree(hContext->getZeHandle(), mappedMemory); + return UR_RESULT_ERROR_UNKNOWN; + } + *phRetMem = reinterpret_cast( + reinterpret_cast(mappedMemory) + offset); + + externalMemoryData->urMemoryHandle = + reinterpret_cast(*phRetMem); + + return UR_RESULT_SUCCESS; +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/image_common.hpp b/unified-runtime/source/adapters/level_zero/image_common.hpp new file mode 100644 index 0000000000000..92778433c206d --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/image_common.hpp @@ -0,0 +1,64 @@ +//===--------- image_common.hpp - Level Zero Adapter +//--------------------===// +// +// Copyright (C) 2024-2025 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +#pragma once + +#include +#include +#include + +#include "v2/common.hpp" + +struct ur_bindless_mem_handle_t { + // Constructor for bindless image handle + ur_bindless_mem_handle_t(ze_image_handle_t zeImage, + const ZeStruct &zeImageDesc) + : zeImage(zeImage) { + + format = zeImageDesc.format; + width = zeImageDesc.width; + height = zeImageDesc.height; + depth = zeImageDesc.depth; + }; + + ze_image_handle_t getZeImage() const { return zeImage.get(); } + + ze_image_format_t getFormat() const { return format; } + uint64_t getWidth() const { return width; } + uint64_t getHeight() const { return height; } + uint64_t getDepth() const { return depth; } + +private: + v2::raii::ze_image_handle_t zeImage; + ze_image_format_t format; + uint64_t width; + uint64_t height; + uint64_t depth; +}; + +/// Construct ZE image desc from UR image format and desc. +ur_result_t ur2zeImageDesc(const ur_image_format_t *ImageFormat, + const ur_image_desc_t *ImageDesc, + ZeStruct &ZeImageDesc); + +ur_result_t getImageRegionHelper(ze_image_desc_t ZeImageDesc, + ur_rect_offset_t *Origin, + ur_rect_region_t *Region, + ze_image_region_t &ZeRegion); + +ur_result_t bindlessImagesHandleCopyFlags( + const void *pSrc, void *pDst, const ur_image_desc_t *pSrcImageDesc, + const ur_image_desc_t *pDstImageDesc, + const ur_image_format_t *pSrcImageFormat, + const ur_image_format_t *pDstImageFormat, + ur_exp_image_copy_region_t *pCopyRegion, + ur_exp_image_copy_flags_t imageCopyFlags, + ze_command_list_handle_t ZeCommandList, ze_event_handle_t zeSignalEvent, + uint32_t numWaitEvents, ze_event_handle_t *phWaitEvents); diff --git a/unified-runtime/source/adapters/level_zero/memory.cpp b/unified-runtime/source/adapters/level_zero/memory.cpp index d68fdc027ba29..82333381a7ceb 100644 --- a/unified-runtime/source/adapters/level_zero/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/memory.cpp @@ -15,8 +15,8 @@ #include "context.hpp" #include "event.hpp" -#include "helpers/image_helpers.hpp" #include "helpers/memory_helpers.hpp" +#include "image_common.hpp" #include "logger/ur_logger.hpp" #include "queue.hpp" #include "ur_interface_loader.hpp" @@ -2321,6 +2321,8 @@ ur_result_t ur_mem_handle_t_::getZeHandle(char *&ZeHandle, access_mode_t mode, case ur_mem_handle_t_::buffer: return reinterpret_cast(this)->getBufferZeHandle( ZeHandle, mode, Device, phWaitEvents, numWaitEvents); + default: + die("ur_mem_handle_t_::getZeHandle: Unhandled memory type"); } ur::unreachable(); } @@ -2335,6 +2337,8 @@ ur_result_t ur_mem_handle_t_::getZeHandlePtr( case ur_mem_handle_t_::buffer: return reinterpret_cast(this)->getBufferZeHandlePtr( ZeHandlePtr, mode, Device, phWaitEvents, numWaitEvents); + default: + die("ur_mem_handle_t_::getZeHandle: Unhandled memory type"); } ur::unreachable(); } diff --git a/unified-runtime/source/adapters/level_zero/memory.hpp b/unified-runtime/source/adapters/level_zero/memory.hpp index 3014810a85a00..7cbc57f041231 100644 --- a/unified-runtime/source/adapters/level_zero/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/memory.hpp @@ -9,7 +9,6 @@ //===----------------------------------------------------------------------===// #pragma once -#include "common.hpp" #include #include #include @@ -19,12 +18,12 @@ #include #include -#include -#include -#include -#include - -#include "ur_level_zero.hpp" +#include "common.hpp" +#include "context.hpp" +#include "event.hpp" +#include "program.hpp" +#include "queue.hpp" +#include "sampler.hpp" struct ur_device_handle_t_; diff --git a/unified-runtime/source/adapters/level_zero/v2/api.cpp b/unified-runtime/source/adapters/level_zero/v2/api.cpp index 1912434529127..bad093fab0cfa 100644 --- a/unified-runtime/source/adapters/level_zero/v2/api.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/api.cpp @@ -41,135 +41,6 @@ ur_result_t urEventSetCallback(ur_event_handle_t hEvent, return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } -ur_result_t urUSMPitchedAllocExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - const ur_usm_desc_t *pUSMDesc, - ur_usm_pool_handle_t pool, size_t widthInBytes, - size_t height, size_t elementSizeBytes, - void **ppMem, size_t *pResultPitch) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesUnsampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_native_handle_t hImage) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesSampledImageHandleDestroyExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_native_handle_t hImage) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesImageAllocateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_image_mem_native_handle_t *phImageMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t -urBindlessImagesImageFreeExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesUnsampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_image_native_handle_t *phImage) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesSampledImageCreateExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_sampler_handle_t hSampler, ur_exp_image_native_handle_t *phImage) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesImageGetInfoExp( - ur_context_handle_t hContext, ur_exp_image_mem_native_handle_t hImageMem, - ur_image_info_t propName, void *pPropValue, size_t *pPropSizeRet) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesMipmapGetLevelExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hImageMem, uint32_t mipmapLevel, - ur_exp_image_mem_native_handle_t *phImageMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t -urBindlessImagesMipmapFreeExp(ur_context_handle_t hContext, - ur_device_handle_t hDevice, - ur_exp_image_mem_native_handle_t hMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesImportExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, size_t size, - ur_exp_external_mem_type_t memHandleType, - ur_exp_external_mem_desc_t *pExternalMemDesc, - ur_exp_external_mem_handle_t *phExternalMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesMapExternalArrayExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - const ur_image_format_t *pImageFormat, const ur_image_desc_t *pImageDesc, - ur_exp_external_mem_handle_t hExternalMem, - ur_exp_image_mem_native_handle_t *phImageMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesMapExternalLinearMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, uint64_t offset, - uint64_t size, ur_exp_external_mem_handle_t hExternalMem, void **ppRetMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesReleaseExternalMemoryExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_mem_handle_t hExternalMem) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesImportExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_semaphore_type_t semHandleType, - ur_exp_external_semaphore_desc_t *pExternalSemaphoreDesc, - ur_exp_external_semaphore_handle_t *phExternalSemaphore) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - -ur_result_t urBindlessImagesReleaseExternalSemaphoreExp( - ur_context_handle_t hContext, ur_device_handle_t hDevice, - ur_exp_external_semaphore_handle_t hExternalSemaphore) { - logger::error("{} function not implemented!", __FUNCTION__); - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; -} - ur_result_t urCommandBufferUpdateKernelLaunchExp( ur_exp_command_buffer_handle_t hCommandBuffer, uint32_t numKernelUpdates, const ur_exp_command_buffer_update_kernel_launch_desc_t diff --git a/unified-runtime/source/adapters/level_zero/v2/common.hpp b/unified-runtime/source/adapters/level_zero/v2/common.hpp index 5d33af7eca6a9..5f26108207fef 100644 --- a/unified-runtime/source/adapters/level_zero/v2/common.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/common.hpp @@ -27,12 +27,12 @@ namespace { namespace v2 { -DECLARE_DESTROY_FUNCTION(zeKernelDestroy); -DECLARE_DESTROY_FUNCTION(zeEventDestroy); -DECLARE_DESTROY_FUNCTION(zeEventPoolDestroy); -DECLARE_DESTROY_FUNCTION(zeContextDestroy); -DECLARE_DESTROY_FUNCTION(zeCommandListDestroy); -DECLARE_DESTROY_FUNCTION(zeImageDestroy); +DECLARE_DESTROY_FUNCTION(zeKernelDestroy) +DECLARE_DESTROY_FUNCTION(zeEventDestroy) +DECLARE_DESTROY_FUNCTION(zeEventPoolDestroy) +DECLARE_DESTROY_FUNCTION(zeContextDestroy) +DECLARE_DESTROY_FUNCTION(zeCommandListDestroy) +DECLARE_DESTROY_FUNCTION(zeImageDestroy) namespace raii { template diff --git a/unified-runtime/source/adapters/level_zero/v2/image.cpp b/unified-runtime/source/adapters/level_zero/v2/image.cpp new file mode 100644 index 0000000000000..3a2af83458ff9 --- /dev/null +++ b/unified-runtime/source/adapters/level_zero/v2/image.cpp @@ -0,0 +1,33 @@ +//===--------- image.cpp - Level Zero Adapter -----------------------------===// +// +// Copyright (C) 2023 Intel Corporation +// +// Part of the Unified-Runtime Project, under the Apache License v2.0 with LLVM +// Exceptions. See LICENSE.TXT +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "common.hpp" + +#include "../image_common.hpp" +#include "../ur_interface_loader.hpp" +#include "../v2/context.hpp" +#include "../v2/memory.hpp" +#include "logger/ur_logger.hpp" +#include "queue_api.hpp" +#include "queue_handle.hpp" + +namespace ur::level_zero { + +ur_result_t +urBindlessImagesImageFreeExp([[maybe_unused]] ur_context_handle_t hContext, + [[maybe_unused]] ur_device_handle_t hDevice, + ur_exp_image_mem_native_handle_t hImageMem) { + ur_bindless_mem_handle_t *urImg = + reinterpret_cast(hImageMem); + delete urImg; + return UR_RESULT_SUCCESS; +} + +} // namespace ur::level_zero diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.cpp b/unified-runtime/source/adapters/level_zero/v2/memory.cpp index 5f8d9c1352735..1f9a3287b51cc 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.cpp @@ -12,8 +12,8 @@ #include "../ur_interface_loader.hpp" #include "context.hpp" -#include "../helpers/image_helpers.hpp" #include "../helpers/memory_helpers.hpp" +#include "../image_common.hpp" static ur_mem_buffer_t::device_access_mode_t getDeviceAccessMode(ur_mem_flags_t memFlag) { diff --git a/unified-runtime/source/adapters/level_zero/v2/memory.hpp b/unified-runtime/source/adapters/level_zero/v2/memory.hpp index fbf134d1432ed..8f54ebd550884 100644 --- a/unified-runtime/source/adapters/level_zero/v2/memory.hpp +++ b/unified-runtime/source/adapters/level_zero/v2/memory.hpp @@ -15,6 +15,8 @@ #include #include "../device.hpp" +#include "../helpers/memory_helpers.hpp" +#include "../image_common.hpp" #include "common.hpp" using usm_unique_ptr_t = std::unique_ptr>; diff --git a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp index 62f815eecc4cb..696d034cfdaf3 100644 --- a/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp +++ b/unified-runtime/source/adapters/level_zero/v2/queue_immediate_in_order.cpp @@ -16,6 +16,8 @@ #include "../common/latency_tracker.hpp" #include "../helpers/kernel_helpers.hpp" +#include "../image_common.hpp" + #include "../program.hpp" #include "../ur_interface_loader.hpp" @@ -729,17 +731,26 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueUSMFreeExp( } ur_result_t ur_queue_immediate_in_order_t::bindlessImagesImageCopyExp( - const void * /*pSrc*/, void * /*pDst*/, - const ur_image_desc_t * /*pSrcImageDesc*/, - const ur_image_desc_t * /*pDstImageDesc*/, - const ur_image_format_t * /*pSrcImageFormat*/, - const ur_image_format_t * /*pDstImageFormat*/, - ur_exp_image_copy_region_t * /*pCopyRegion*/, - ur_exp_image_copy_flags_t /*imageCopyFlags*/, - uint32_t /*numEventsInWaitList*/, - const ur_event_handle_t * /*phEventWaitList*/, - ur_event_handle_t * /*phEvent*/) { - return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; + const void *pSrc, void *pDst, const ur_image_desc_t *pSrcImageDesc, + const ur_image_desc_t *pDstImageDesc, + const ur_image_format_t *pSrcImageFormat, + const ur_image_format_t *pDstImageFormat, + ur_exp_image_copy_region_t *pCopyRegion, + ur_exp_image_copy_flags_t imageCopyFlags, uint32_t numEventsInWaitList, + const ur_event_handle_t *phEventWaitList, ur_event_handle_t *phEvent) { + + auto commandListMgr = commandListManager.lock(); + + auto zeSignalEvent = + getSignalEvent(commandListMgr, phEvent, UR_COMMAND_MEM_IMAGE_COPY); + auto waitListView = + getWaitListView(commandListMgr, phEventWaitList, numEventsInWaitList); + + return bindlessImagesHandleCopyFlags( + pSrc, pDst, pSrcImageDesc, pDstImageDesc, pSrcImageFormat, + pDstImageFormat, pCopyRegion, imageCopyFlags, + commandListMgr->getZeCommandList(), zeSignalEvent, waitListView.num, + waitListView.handles); } ur_result_t @@ -908,6 +919,10 @@ ur_result_t ur_queue_immediate_in_order_t::enqueueNativeCommandExp( ur_exp_enqueue_native_command_function_t, void *, uint32_t, const ur_mem_handle_t *, const ur_exp_enqueue_native_command_properties_t *, uint32_t, const ur_event_handle_t *, ur_event_handle_t *) { + logger::error( + logger::LegacyMessage("[UR][L0_v2] {} function not implemented!"), + "{} function not implemented!", __FUNCTION__); + return UR_RESULT_ERROR_UNSUPPORTED_FEATURE; } } // namespace v2