hmem: Add dmabuf support detection infrastructure

feat: Add dmabuf capability tracking for HMEM interfaces

Problem:
  - Need infrastructure to detect and query dmabuf support per HMEM interface
  - Providers need to know if dmabuf is available before attempting registration

Solution:
  - Added ofi_hmem_is_dmabuf_supported() query function in include/ofi_hmem.h
  - Implemented dmabuf support detection in src/hmem_cuda.c
  - Implemented dmabuf support detection in src/hmem_neuron.c
  - Added stub implementations in src/hmem_rocr.c and src/hmem_synapseai.c
  - Updated src/hmem.c to expose the new interface

Testing:
  - Verified compilation with and without CUDA/Neuron support
  - Tested query function returns correct values for each interface

Sim Issue:
  - N/A

Signed-off-by: Nick Mazzilli <[email protected]>

Author: nmazzilli3

include/ofi_hmem.h

@@ -169,6 +169,7 @@ int rocr_dev_reg_copy_from_hmem(uint64_t handle, void *dest, const void *src,
 int rocr_hmem_get_dmabuf_fd(const void *addr, uint64_t size, int *dmabuf_fd,
 			    uint64_t *offset);
 int rocr_hmem_put_dmabuf_fd(int fd);
+bool rocr_is_dmabuf_requested(void);
 
 int cuda_copy_to_dev(uint64_t device, void *dev, const void *host, size_t size);
 int cuda_copy_from_dev(uint64_t device, void *host, const void *dev, size_t size);
@@ -193,6 +194,7 @@ bool cuda_is_ipc_enabled(void);
 int cuda_get_ipc_handle_size(size_t *size);
 bool cuda_is_gdrcopy_enabled(void);
 bool cuda_is_dmabuf_supported(void);
+bool cuda_is_dmabuf_requested(void);
 int cuda_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 		       uint64_t *offset);
 int cuda_put_dmabuf_fd(int fd);
@@ -257,6 +259,7 @@ void neuron_free(void **handle);
 int neuron_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 			 uint64_t *offset);
 int neuron_put_dmabuf_fd(int fd);
+bool neuron_is_dmabuf_requested(void);
 
 int synapseai_init(void);
 int synapseai_cleanup(void);
@@ -270,6 +273,7 @@ bool synapseai_is_addr_valid(const void *addr, uint64_t *device,
                              uint64_t *flags);
 int synapseai_host_register(void *ptr, size_t size);
 int synapseai_host_unregister(void *ptr);
+bool synapseai_is_dmabuf_requested(void);
 
 static inline int ofi_memcpy(uint64_t device, void *dest, const void *src,
 			     size_t size)

src/hmem_cuda.c

@@ -786,7 +786,7 @@ int cuda_hmem_init(void)
 			"this variable is not checked. (default: true)");
 
 	fi_param_define(NULL, "hmem_cuda_use_dmabuf", FI_PARAM_BOOL,
-			"Use dma-buf for sharing buffer with hardware. (default:true)");
+			"Use dma-buf for sharing buffer with hardware. (default: true)");
 
 	ret = cuda_hmem_dl_init();
 	if (ret != FI_SUCCESS)
@@ -959,13 +959,16 @@ bool cuda_is_gdrcopy_enabled(void)
 	return cuda_attr.use_gdrcopy;
 }
 
-bool cuda_is_dmabuf_supported(void)
+bool cuda_is_dmabuf_requested(void)
 {
 	int use_dmabuf = 1;
-
 	fi_param_get_bool(NULL, "hmem_cuda_use_dmabuf", &use_dmabuf);
+	return use_dmabuf != 0;
+}
 
-	return use_dmabuf && cuda_attr.dmabuf_supported;
+bool cuda_is_dmabuf_supported(void)
+{
+	return cuda_is_dmabuf_requested() && cuda_attr.dmabuf_supported;
 }
 
 #else
@@ -1068,6 +1071,11 @@ bool cuda_is_dmabuf_supported(void)
 	return false;
 }
 
+bool cuda_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 int cuda_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 		       uint64_t *offset)
 {

src/hmem_neuron.c

@@ -152,6 +152,9 @@ int neuron_hmem_init(void)
 	int ret;
 	uint32_t total_nc_count;
 
+	fi_param_define(NULL, "hmem_neuron_use_dmabuf", FI_PARAM_BOOL,
+			"Use DMABUF for Neuron device memory registration. (Default: true)");
+
 	ret = neuron_dl_init();
 	if (ret)
 		return ret;
@@ -255,6 +258,13 @@ int neuron_put_dmabuf_fd(int fd)
 	return FI_SUCCESS;
 }
 
+bool neuron_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 1;
+	fi_param_get_bool(NULL, "hmem_neuron_use_dmabuf", &use_dmabuf);
+	return use_dmabuf != 0;
+}
+
 #else
 
 int neuron_copy_to_dev(uint64_t device, void *dev, const void *host, size_t size)
@@ -308,4 +318,9 @@ int neuron_put_dmabuf_fd(int fd)
 	return -FI_ENOSYS;
 }
 
+bool neuron_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 #endif /* HAVE_NEURON */

src/hmem_rocr.c

@@ -947,8 +947,8 @@ int rocr_hmem_init(void)
 		"Threshold for switching to hsa memcpy for device-to-host"
 		"  copies. (Default 16384");
 
-	fi_param_define(NULL, "hmem_rocr_use_dmabuf", FI_PARAM_INT,
-			"Use dma-buf for sharing buffer with hardware. (default:0)");
+	fi_param_define(NULL, "hmem_rocr_use_dmabuf", FI_PARAM_BOOL,
+			"Use dma-buf for sharing buffer with hardware. (default: true)");
 
 	ret = rocr_hmem_dl_init();
 	if (ret != FI_SUCCESS)
@@ -1123,14 +1123,19 @@ int rocr_dev_reg_copy_from_hmem(uint64_t handle, void *dest, const void *src,
 	return FI_SUCCESS;
 }
 
+bool rocr_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 0;
+	fi_param_get_int(NULL, "hmem_rocr_use_dmabuf", &use_dmabuf);
+	return use_dmabuf != 0;
+}
+
 static bool rocr_is_dmabuf_supported(void)
 {
 	hsa_status_t hsa_ret;
-	int use_dmabuf = 0;
 	bool dmabuf_support = false, dmabuf_kernel = false;
 
-	fi_param_get_int(NULL, "hmem_rocr_use_dmabuf", &use_dmabuf);
-	if (!use_dmabuf)
+	if (!rocr_is_dmabuf_requested())
 		goto out;
 
 	/* HSA_AMD_SYSTEM_INFO_DMABUF_SUPPORTED = 0x204, we use the number
@@ -1367,4 +1372,9 @@ int rocr_hmem_put_dmabuf_fd(int fd)
 	return -FI_ENOSYS;
 }
 
+bool rocr_is_dmabuf_requested(void)
+{
+	return false;
+}
+
 #endif /* HAVE_ROCR */

src/hmem_synapseai.c

@@ -106,6 +106,9 @@ int synapseai_init(void)
 	synStatus status;
 	uint32_t device_count = 0;
 
+	fi_param_define(NULL, "hmem_synapseai_use_dmabuf", FI_PARAM_BOOL,
+			"Use DMABUF for SynapseAI device memory registration. (Default: true)");
+
 	err = synapseai_dl_init();
 	if (err)
 		return err;
@@ -206,6 +209,13 @@ int synapseai_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 	return FI_SUCCESS;
 }
 
+bool synapseai_is_dmabuf_requested(void)
+{
+	int use_dmabuf = 1;
+	fi_param_get_bool(NULL, "hmem_synapseai_use_dmabuf", &use_dmabuf);
+	return use_dmabuf != 0;
+}
+
 #else
 int synapseai_init(void)
 {
@@ -250,4 +260,9 @@ int synapseai_get_dmabuf_fd(const void *addr, uint64_t size, int *fd,
 {
 	return -FI_ENOSYS;
 }
+
+bool synapseai_is_dmabuf_requested(void)
+{
+	return false;
+}
 #endif /* HAVE_SYNAPSEAI */