hmem: Add dmabuf support detection infrastructure

feat: Add dmabuf capability tracking for HMEM interfaces

Problem:
  - Need infrastructure to detect and query dmabuf support per HMEM
	interface
  - Providers need to know if dmabuf is available before attempting
	registration

Solution:
  - Added ofi_hmem_is_dmabuf_supported() query function in
	include/ofi_hmem.h
  - Implemented dmabuf support detection in src/hmem_cuda.c
  - Implemented dmabuf support detection in src/hmem_neuron.c
  - Added stub implementations in src/hmem_rocr.c and src/hmem_synapseai.c
  - Updated src/hmem.c to expose the new interface
  - Made use_dmabuf environment variable for ROCr true by default to match
	behavior of others

Co-authored-by: Nick Mazzilli <nmazzill@amazon.com>

Signed-off-by: Charles Stoll <stollcha@amazon.com>

Author: charlesstoll

include/ofi_hmem.h

@@ -171,6 +171,7 @@ int rocr_hmem_get_dmabuf_fd(const void *addr, uint64_t size, int *dmabuf_fd,
 int rocr_hmem_put_dmabuf_fd(int fd);
 void *rocr_alloc(size_t size);
 void rocr_free(void *ptr);
+bool rocr_is_dmabuf_requested(void);
 
 int cuda_copy_to_dev(uint64_t device, void *dev, const void *host, size_t size);
 int cuda_copy_from_dev(uint64_t device, void *host, const void *dev, size_t size);
@@ -195,6 +196,7 @@ bool cuda_is_ipc_enabled(void);
 int cuda_get_ipc_handle_size(size_t *size);
 bool cuda_is_gdrcopy_enabled(void);
 bool cuda_is_dmabuf_supported(void);
+bool cuda_is_dmabuf_requested(void);
 int cuda_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 		       uint64_t *offset);
 int cuda_put_dmabuf_fd(int fd);
@@ -247,6 +249,7 @@ int ze_dev_reg_copy_from_hmem(uint64_t handle, void *dest, const void *src,
 			      size_t size);
 int ze_hmem_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 			  uint64_t *offset);
+bool ze_is_dmabuf_requested(void);
 
 int neuron_copy_to_dev(uint64_t device, void *dev, const void *host, size_t size);
 int neuron_copy_from_dev(uint64_t device, void *host, const void *dev, size_t size);
@@ -259,6 +262,7 @@ void neuron_free(void **handle);
 int neuron_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 			 uint64_t *offset);
 int neuron_put_dmabuf_fd(int fd);
+bool neuron_is_dmabuf_requested(void);
 
 int synapseai_init(void);
 int synapseai_cleanup(void);
@@ -272,6 +276,7 @@ bool synapseai_is_addr_valid(const void *addr, uint64_t *device,
                              uint64_t *flags);
 int synapseai_host_register(void *ptr, size_t size);
 int synapseai_host_unregister(void *ptr);
+bool synapseai_is_dmabuf_requested(void);
 
 static inline int ofi_memcpy(uint64_t device, void *dest, const void *src,
 			     size_t size)
@@ -462,5 +467,5 @@ int ofi_hmem_dev_reg_copy_from_hmem(enum fi_hmem_iface iface, uint64_t handle,
 int ofi_hmem_get_dmabuf_fd(enum fi_hmem_iface, const void *addr, uint64_t size,
 			   int *fd, uint64_t *offset);
 int ofi_hmem_put_dmabuf_fd(enum fi_hmem_iface iface, int fd);
-
+bool ofi_hmem_is_dmabuf_env_var_enabled(enum fi_hmem_iface iface);
 #endif /* _OFI_HMEM_H_ */

src/hmem.c

@@ -831,3 +831,32 @@ int ofi_hmem_put_dmabuf_fd(enum fi_hmem_iface iface, int fd)
 {
 	return hmem_ops[iface].put_dmabuf_fd(fd);
 }
+
+/**
+ * @brief Check if DMABUF is enabled for a specific HMEM interface
+ *
+ * This function checks the environment variables to determine if DMABUF
+ * should be used for the specified HMEM interface.
+ *
+ * @param[in] iface The HMEM interface to check
+ * @return true if DMABUF is enabled for the interface, false otherwise
+ */
+bool ofi_hmem_is_dmabuf_env_var_enabled(enum fi_hmem_iface iface)
+{
+	switch (iface) {
+	case FI_HMEM_SYSTEM:
+		return false;
+	case FI_HMEM_CUDA:
+		return cuda_is_dmabuf_requested();
+	case FI_HMEM_NEURON:
+		return neuron_is_dmabuf_requested();
+	case FI_HMEM_SYNAPSEAI:
+		return synapseai_is_dmabuf_requested();
+	case FI_HMEM_ROCR:
+		return rocr_is_dmabuf_requested();
+	case FI_HMEM_ZE:
+		return ze_is_dmabuf_requested();
+	default:
+		return false;
+	}
+}

src/hmem_cuda.c

@@ -786,7 +786,7 @@ int cuda_hmem_init(void)
 			"this variable is not checked. (default: true)");
 
 	fi_param_define(NULL, "hmem_cuda_use_dmabuf", FI_PARAM_BOOL,
-			"Use dma-buf for sharing buffer with hardware. (default:true)");
+			"Use dma-buf for sharing buffer with hardware. (default: true)");
 
 	ret = cuda_hmem_dl_init();
 	if (ret != FI_SUCCESS)
@@ -959,13 +959,17 @@ bool cuda_is_gdrcopy_enabled(void)
 	return cuda_attr.use_gdrcopy;
 }
 
-bool cuda_is_dmabuf_supported(void)
+bool cuda_is_dmabuf_requested(void)
 {
 	int use_dmabuf = 1;
 
 	fi_param_get_bool(NULL, "hmem_cuda_use_dmabuf", &use_dmabuf);
+	return use_dmabuf;
+}
 
-	return use_dmabuf && cuda_attr.dmabuf_supported;
+bool cuda_is_dmabuf_supported(void)
+{
+	return cuda_is_dmabuf_requested() && cuda_attr.dmabuf_supported;
 }
 
 #else
@@ -1068,6 +1072,11 @@ bool cuda_is_dmabuf_supported(void)
 	return false;
 }
 
+bool cuda_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 int cuda_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 		       uint64_t *offset)
 {

src/hmem_neuron.c

@@ -152,6 +152,9 @@ int neuron_hmem_init(void)
 	int ret;
 	uint32_t total_nc_count;
 
+	fi_param_define(NULL, "hmem_neuron_use_dmabuf", FI_PARAM_BOOL,
+			"Use DMABUF for Neuron device memory registration. (Default: true)");
+
 	ret = neuron_dl_init();
 	if (ret)
 		return ret;
@@ -255,6 +258,14 @@ int neuron_put_dmabuf_fd(int fd)
 	return FI_SUCCESS;
 }
 
+bool neuron_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 1;
+
+	fi_param_get_bool(NULL, "hmem_neuron_use_dmabuf", &use_dmabuf);
+	return use_dmabuf;
+}
+
 #else
 
 int neuron_copy_to_dev(uint64_t device, void *dev, const void *host, size_t size)
@@ -308,4 +319,9 @@ int neuron_put_dmabuf_fd(int fd)
 	return -FI_ENOSYS;
 }
 
+bool neuron_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 #endif /* HAVE_NEURON */

src/hmem_rocr.c

@@ -1081,8 +1081,8 @@ int rocr_hmem_init(void)
 		"Threshold for switching to hsa memcpy for device-to-host"
 		"  copies. (Default 16384");
 
-	fi_param_define(NULL, "hmem_rocr_use_dmabuf", FI_PARAM_INT,
-			"Use dma-buf for sharing buffer with hardware. (default:0)");
+	fi_param_define(NULL, "hmem_rocr_use_dmabuf", FI_PARAM_BOOL,
+			"Use dma-buf for sharing buffer with hardware. (default: true)");
 
 	ret = rocr_hmem_dl_init();
 	if (ret != FI_SUCCESS)
@@ -1257,14 +1257,20 @@ int rocr_dev_reg_copy_from_hmem(uint64_t handle, void *dest, const void *src,
 	return FI_SUCCESS;
 }
 
+bool rocr_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 1;
+
+	fi_param_get_int(NULL, "hmem_rocr_use_dmabuf", &use_dmabuf);
+	return use_dmabuf;
+}
+
 static bool rocr_is_dmabuf_supported(void)
 {
 	hsa_status_t hsa_ret;
-	int use_dmabuf = 0;
 	bool dmabuf_support = false, dmabuf_kernel = false;
 
-	fi_param_get_int(NULL, "hmem_rocr_use_dmabuf", &use_dmabuf);
-	if (!use_dmabuf)
+	if (!rocr_is_dmabuf_requested())
 		goto out;
 
 	/* HSA_AMD_SYSTEM_INFO_DMABUF_SUPPORTED = 0x204, we use the number
@@ -1511,4 +1517,9 @@ void rocr_free(void *ptr)
 	return;
 }
 
+bool rocr_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 #endif /* HAVE_ROCR */

src/hmem_synapseai.c

@@ -106,6 +106,9 @@ int synapseai_init(void)
 	synStatus status;
 	uint32_t device_count = 0;
 
+	fi_param_define(NULL, "hmem_synapseai_use_dmabuf", FI_PARAM_BOOL,
+			"Use DMABUF for SynapseAI device memory registration. (Default: true)");
+
 	err = synapseai_dl_init();
 	if (err)
 		return err;
@@ -206,6 +209,14 @@ int synapseai_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 	return FI_SUCCESS;
 }
 
+bool synapseai_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 1;
+
+	fi_param_get_bool(NULL, "hmem_synapseai_use_dmabuf", &use_dmabuf);
+	return use_dmabuf;
+}
+
 #else
 int synapseai_init(void)
 {
@@ -250,4 +261,9 @@ int synapseai_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 {
 	return -FI_ENOSYS;
 }
+
+bool synapseai_is_dmabuf_requested(void)
+{
+	return false;
+}
 #endif /* HAVE_SYNAPSEAI */

src/hmem_ze.c

@@ -681,6 +681,9 @@ int ze_hmem_init(void)
 	if (enginestr)
 		sscanf(enginestr, "%d.%d", &ordinal, &index);
 
+	fi_param_define(NULL, "hmem_ze_use_dmabuf", FI_PARAM_BOOL,
+			"Use DMABUF for ZE device memory registration. (Default: true)");
+
 	ret = ze_hmem_dl_init();
 	if (ret)
 		return ret;
@@ -1088,6 +1091,14 @@ int ze_hmem_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 	return 0;
 }
 
+bool ze_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 1;
+
+	fi_param_get_int(NULL, "hmem_ze_use_dmabuf", &use_dmabuf);
+	return use_dmabuf;
+}
+
 struct ze_dev_reg_handle {
 	void *base_dev;
 	void *base_host;
@@ -1349,4 +1360,9 @@ int ze_hmem_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 	return -FI_ENOSYS;
 }
 
+bool ze_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 #endif /* HAVE_ZE */