config: Enable DMA-BUF by default (except old EFA)

Change the default value of OFI_NCCL_DISABLE_DMABUF to 0 so that
DMA-BUF is enabled by default on platforms that have Libfabric 1.20,
kernel 5.12 or later, and CUDA 11.7 with DMA-BUF support.

At the same time, disable DMA-BUF support on 1st, 2nd, and 3rd
generation EFA, due to an issue with operations using very
large page entry count MRs and an issue with page merging in the
RDMA subsystem (see comment in nccl_ofi_net.c for more details).

Signed-off-by: Arun Karthik <[email protected]>
Signed-off-by: Brian Barrett <[email protected]>

Author: akkart-aws

include/nccl_ofi_param.h

@@ -273,9 +273,9 @@ OFI_NCCL_PARAM_INT(disable_gdr_required_check, "DISABLE_GDR_REQUIRED_CHECK", 0);
  * the plugin has no freedom to renegotiate DMABUF support with NCCL, and so it
  * is fatal. Under those conditions, users should ensure that they have set this
  * environment variable to '1' to force NCCL to avoid providing dmabuf file
- * desciptors. This is the default, pending perf investigations.
+ * desciptors.
  */
-OFI_NCCL_PARAM_INT(disable_dmabuf, "DISABLE_DMABUF", 1);
+OFI_NCCL_PARAM_INT(disable_dmabuf, "DISABLE_DMABUF", 0);
 
 /*
  * Messages sized larger than this threshold will be striped across multiple rails

src/nccl_ofi_net.cpp

@@ -578,11 +578,42 @@ int nccl_net_ofi_info_properties(nccl_net_ofi_plugin_t *plugin, struct fi_info *
 
 	props->max_mr_key_size = nic_prov->domain_attr->mr_key_size;
 
-
 	props->dmabuf_support = ((nic_prov->caps & FI_HMEM) != 0) &&
 		FI_VERSION_GE(nic_prov->fabric_attr->api_version, FI_VERSION(1, 20)) &&
 		nccl_ofi_dmabuf_viable()
 		;
+	if (props->dmabuf_support && strncmp("efa", nic_prov->fabric_attr->prov_name, strlen("efa")) == 0) {
+		// Generations 1-3 of EFA have a firmware issue that can result
+		// in communication failures with MRs that cover a large number
+		// of page entries.  This is not usually a problem, because page
+		// merging greatly reduces the number of page entries in the MR.
+		// However, the RDMA subsystem in the Linux kernel did not
+		// properly execute page merging for dmabuf entries until a
+		// recent patch
+		// (https://web.git.kernel.org/pub/scm/linux/kernel/git/rdma/rdma.git/commit/?id=486055f5e09df9),
+		// and the lack of page merging increased the probability of
+		// hitting the EFA issue.  Testing for the fixed kernel version
+		// is effectively impossible (the issue can also be fixed in the
+		// EFA kmod itself, and backports are likely, so a simple kernel
+		// version check is insufficient), so instead we only support
+		// dmabuf by default in Generation 4 of EFA.  When the
+		// communication failure issue is resolved in previous
+		// generations, this code will be removed and dmabuf will be
+		// available by default everywhere.
+		if (nic_prov->nic == NULL || nic_prov->nic->device_attr == NULL) {
+			NCCL_OFI_TRACE(NCCL_INIT | NCCL_NET,
+				       "DMA-BUF disabled due to missing nic data");
+			props->dmabuf_support = false;
+		} else if (strcmp("efa0", nic_prov->nic->device_attr->device_id) == 0 ||
+			   strcmp("efa1", nic_prov->nic->device_attr->device_id) == 0 ||
+			   strcmp("efa2", nic_prov->nic->device_attr->device_id) == 0) {
+			NCCL_OFI_TRACE(NCCL_INIT | NCCL_NET,
+				       "DMA-BUF disabled due to EFA device id %s",
+				       nic_prov->nic->device_attr->device_id);
+			props->dmabuf_support = false;
+		}
+	}
+
 	if (props->dmabuf_support) {
 		NCCL_OFI_TRACE(NCCL_INIT | NCCL_NET, "DMA-BUF support is advertised in properties.");
 	}