UCT/ZE: Fix reset path, DMA-BUF ownership, and descriptor init (openucx#11223)

* UCT/ZE: Add device topology registration

Implement Level Zero device enumeration and topology registration
to properly integrate Intel GPUs with UCX's topology subsystem.

Key changes:
- Enumerate Level Zero devices and sub-devices during initialization
- Register each physical device once with topology using PCI bus ID
- All sub-devices on same device share parent's sys_dev for IB affinity
- Device naming: "GPU0" for single sub-device, "GPU0.0"/"GPU0.1" for multi
- Use zeDevicePciGetPropertiesExt() for PCI properties (Level Zero 1.0+ compat)
- Enable auxiliary paths for multi-path routing

Architecture:
- Static sub-device array populated at init, read-only after
- Query functions return empty list on init failure (not error)
- One MD resource, one TL device per sub-device

API cleanup:
- Removed unused functions from public header

* UCT/ZE: Fix code style in ze_base files

* UCT/ZE: Fix topology registration for flat device hierarchies

Fix device enumeration on systems where Level Zero reports tiles as
separate root devices (e.g., Ponte Vecchio Data Center Max) rather
than hierarchical sub-devices.

Changes:
- Detect duplicate PCI addresses (BDF) to identify tiles on same GPU
- Share sys_dev across root devices with identical PCI address
- Support both hierarchical (zeDeviceGetSubDevices) and flat models
- Preserve all 8 device handles (GPU0-GPU7) with correct 4-sys_dev mapping

Fixes incorrect NUMA/IB affinity when flat hierarchy causes separate
topology registration for tiles on same physical device.

* UCX/ZE: Refactor base initialization into helper functions

* UCT/ZE/COPY: always reset command list and propagate reset failures

* UCT/ZE/COPY: Close exported dmabuf fd after dup in mem_query

zeMemGetAllocProperties returns an exported dmabuf fd that must be
closed by UCX after duplicating it for the caller. Previously, each
mem_query leaked one fd.

Add a centralized cleanup path to always close the original fd and
handle dup() failure.

* UCT/ZE/COPY: initialize stype in Level Zero alloc descriptors

Set mandatory stype in ze_host_mem_alloc_desc_t and
ze_device_mem_alloc_desc_t used by mem_alloc.

Although the descriptors were zero-initialized, explicit stype is
required by Level Zero and improves compatibility with stricter
runtime validation and future extension chaining.

* UCT/ZE/COPY: remove redundant ep_create/ep_destroy ops entries

* UCT/ZE: style and whitespace cleanup

* UCT/ZE/COPY: preserve Level Zero DMA-BUF export fd ownership in mem_query

* UCT/ZE/COPY: clang-format cleanup in ZE copy files

* UCT/ZE/COPY: simplify dmabuf fd setup in mem_query

Author: yafshar

src/uct/ze/copy/ze_copy_ep.c

@@ -20,6 +20,7 @@
 
 #include <level_zero/ze_api.h>
 
+
 static UCS_CLASS_INIT_FUNC(uct_ze_copy_ep_t, const uct_ep_params_t *params)
 {
     uct_ze_copy_iface_t *iface = ucs_derived_of(params->iface,
@@ -45,6 +46,7 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr,
     size_t size                = uct_iov_get_length(iov);
     uct_ze_copy_iface_t *iface = ucs_derived_of(tl_ep->iface,
                                                 uct_ze_copy_iface_t);
+    ucs_status_t status        = UCS_OK;
     ze_result_t ret;
     void *src, *dst;
 
@@ -62,33 +64,40 @@ ucs_status_t uct_ze_copy_ep_zcopy(uct_ep_h tl_ep, uint64_t remote_addr,
     ret = zeCommandListAppendMemoryCopy(iface->ze_cmdl, dst, src, size, NULL, 0,
                                         NULL);
     if (ret != ZE_RESULT_SUCCESS) {
-        return UCS_ERR_IO_ERROR;
+        status = UCS_ERR_IO_ERROR;
+        goto out_reset;
     }
 
     ret = zeCommandListClose(iface->ze_cmdl);
     if (ret != ZE_RESULT_SUCCESS) {
-        return UCS_ERR_IO_ERROR;
+        status = UCS_ERR_IO_ERROR;
+        goto out_reset;
     }
 
     ret = zeCommandQueueExecuteCommandLists(iface->ze_cmdq, 1, &iface->ze_cmdl,
                                             NULL);
     if (ret != ZE_RESULT_SUCCESS) {
-        return UCS_ERR_IO_ERROR;
+        status = UCS_ERR_IO_ERROR;
+        goto out_reset;
     }
 
     ret = zeCommandQueueSynchronize(iface->ze_cmdq, UINT64_MAX);
     if (ret != ZE_RESULT_SUCCESS) {
-        return UCS_ERR_IO_ERROR;
+        status = UCS_ERR_IO_ERROR;
     }
 
+out_reset:
     ret = zeCommandListReset(iface->ze_cmdl);
     if (ret != ZE_RESULT_SUCCESS) {
-        return UCS_ERR_IO_ERROR;
+        ucs_error("zeCommandListReset failed: 0x%x", ret);
+        if (status == UCS_OK) {
+            status = UCS_ERR_IO_ERROR;
+        }
     }
 
     ucs_trace("ze memory copy from src %p to dst %p, len %ld", src, dst, size);
 
-    return UCS_OK;
+    return status;
 }
 
 ucs_status_t uct_ze_copy_ep_get_zcopy(uct_ep_h tl_ep, const uct_iov_t *iov,

src/uct/ze/copy/ze_copy_iface.c

@@ -139,22 +139,26 @@ static uct_iface_ops_t uct_ze_copy_iface_ops = {
     .ep_put_short             = uct_ze_copy_ep_put_short,
     .ep_get_zcopy             = uct_ze_copy_ep_get_zcopy,
     .ep_put_zcopy             = uct_ze_copy_ep_put_zcopy,
-    .ep_pending_add           = (uct_ep_pending_add_func_t)ucs_empty_function_return_busy,
-    .ep_pending_purge         = (uct_ep_pending_purge_func_t)ucs_empty_function,
+    .ep_pending_add           = (uct_ep_pending_add_func_t)
+            ucs_empty_function_return_busy,
+    .ep_pending_purge         = (uct_ep_pending_purge_func_t)
+            ucs_empty_function,
     .ep_flush                 = uct_base_ep_flush,
     .ep_fence                 = uct_base_ep_fence,
-    .ep_create                = uct_ep_create,
-    .ep_destroy               = uct_ep_destroy,
     .ep_create                = UCS_CLASS_NEW_FUNC_NAME(uct_ze_copy_ep_t),
     .ep_destroy               = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_copy_ep_t),
     .iface_flush              = uct_base_iface_flush,
     .iface_fence              = uct_base_iface_fence,
-    .iface_progress_enable    = (uct_iface_progress_enable_func_t)ucs_empty_function,
-    .iface_progress_disable   = (uct_iface_progress_disable_func_t)ucs_empty_function,
-    .iface_progress           = (uct_iface_progress_func_t)ucs_empty_function_return_zero,
+    .iface_progress_enable    = (uct_iface_progress_enable_func_t)
+            ucs_empty_function,
+    .iface_progress_disable   = (uct_iface_progress_disable_func_t)
+            ucs_empty_function,
+    .iface_progress           = (uct_iface_progress_func_t)
+            ucs_empty_function_return_zero,
     .iface_close              = UCS_CLASS_DELETE_FUNC_NAME(uct_ze_copy_iface_t),
     .iface_query              = uct_ze_copy_iface_query,
-    .iface_get_device_address = (uct_iface_get_device_address_func_t)ucs_empty_function_return_success,
+    .iface_get_device_address = (uct_iface_get_device_address_func_t)
+            ucs_empty_function_return_success,
     .iface_get_address        = uct_ze_copy_iface_get_address,
     .iface_is_reachable       = uct_base_iface_is_reachable,
 };

src/uct/ze/copy/ze_copy_md.c

@@ -66,9 +66,13 @@ uct_ze_copy_mem_alloc(uct_md_h tl_md, size_t *length_p, void **address_p,
                       ucs_memory_type_t mem_type, ucs_sys_device_t sys_dev,
                       unsigned flags, const char *alloc_name, uct_mem_h *memh_p)
 {
-    uct_ze_copy_md_t *md = ucs_derived_of(tl_md, uct_ze_copy_md_t);
-    ze_host_mem_alloc_desc_t host_desc  = {};
-    ze_device_mem_alloc_desc_t dev_desc = {};
+    uct_ze_copy_md_t *md                = ucs_derived_of(tl_md, uct_ze_copy_md_t);
+    ze_host_mem_alloc_desc_t host_desc  = {
+        .stype = ZE_STRUCTURE_TYPE_HOST_MEM_ALLOC_DESC
+    };
+    ze_device_mem_alloc_desc_t dev_desc = {
+        .stype = ZE_STRUCTURE_TYPE_DEVICE_MEM_ALLOC_DESC
+    };
     size_t alignment                    = ucs_get_page_size();
     ucs_status_t status;
 
@@ -141,10 +145,11 @@ static ucs_status_t
 uct_ze_copy_md_query_attributes(uct_md_h md, const void *addr, size_t length,
                                 ucs_memory_info_t *mem_info, int *dmabuf_fd)
 {
-    uct_ze_copy_md_t *ze_md = ucs_derived_of(md, uct_ze_copy_md_t);
+    uct_ze_copy_md_t *ze_md                  = ucs_derived_of(md, uct_ze_copy_md_t);
     ze_external_memory_export_fd_t export_fd = {
         .stype = ZE_STRUCTURE_TYPE_EXTERNAL_MEMORY_EXPORT_FD,
-        .flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF
+        .flags = ZE_EXTERNAL_MEMORY_TYPE_FLAG_DMA_BUF,
+        .fd    = UCT_DMABUF_FD_INVALID
     };
     ze_memory_allocation_properties_t props  = {
         .stype = ZE_STRUCTURE_TYPE_MEMORY_ALLOCATION_PROPERTIES,
@@ -185,12 +190,18 @@ static ucs_status_t uct_ze_copy_md_mem_query(uct_md_h md, const void *addr,
                                              const size_t length,
                                              uct_md_mem_attr_t *mem_attr_p)
 {
-    int dmabuf_fd = UCT_DMABUF_FD_INVALID;
-    ucs_status_t status;
+    int dmabuf_fd    = UCT_DMABUF_FD_INVALID;
+    int *dmabuf_fd_p = NULL;
     ucs_memory_info_t mem_info;
+    ucs_status_t status;
+
+    if (mem_attr_p->field_mask & UCT_MD_MEM_ATTR_FIELD_DMABUF_FD) {
+        mem_attr_p->dmabuf_fd = UCT_DMABUF_FD_INVALID;
+        dmabuf_fd_p           = &dmabuf_fd;
+    }
 
     status = uct_ze_copy_md_query_attributes(md, addr, length, &mem_info,
-                                             &dmabuf_fd);
+                                             dmabuf_fd_p);
     if (status != UCS_OK) {
         return status;
     }
@@ -215,13 +226,26 @@ static ucs_status_t uct_ze_copy_md_mem_query(uct_md_h md, const void *addr,
     }
 
     if (mem_attr_p->field_mask & UCT_MD_MEM_ATTR_FIELD_DMABUF_FD) {
+        if (dmabuf_fd == UCT_DMABUF_FD_INVALID) {
+            return UCS_ERR_UNSUPPORTED;
+        }
+
         mem_attr_p->dmabuf_fd = dup(dmabuf_fd);
+        if (mem_attr_p->dmabuf_fd < 0) {
+            return UCS_ERR_IO_ERROR;
+        }
+
+        /* NOTE: Do not close(dmabuf_fd) here. Level Zero caches this fd
+         * internally per allocation. Closing it can invalidate the cache and
+         * lead to fd reuse conflicts with other transports.
+         */
     }
 
     if (mem_attr_p->field_mask & UCT_MD_MEM_ATTR_FIELD_DMABUF_OFFSET) {
         mem_attr_p->dmabuf_offset = UCS_PTR_BYTE_DIFF(mem_info.base_address,
                                                       addr);
     }
+
     return UCS_OK;
 }
 
@@ -330,4 +354,3 @@ uct_component_t uct_ze_copy_component = {
     .md_vfs_init    = (uct_component_md_vfs_init_func_t)ucs_empty_function
 };
 UCT_COMPONENT_REGISTER(&uct_ze_copy_component);
-