Add nccl_env_vars file

Author: msimberg

docs/software/communication/nccl_env_vars

@@ -0,0 +1,20 @@
+# This forces NCCL to use the libfabric plugin, enabling full use of the
+# Slingshot network. If the plugin can not be found, applications will fail to
+# start. With the default value, applications would instead fall back to e.g.
+# TCP, which would be significantly slower than with the plugin. More information
+# about `NCCL_NET` can be found at
+# https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-net.
+export NCCL_NET="AWS Libfabric"
+# Use GPU Direct RDMA when GPU and NIC are on the same NUMA node. More
+# information about `NCCL_NET_GDR_LEVEL` can be found at
+# https://docs.nvidia.com/deeplearning/nccl/user-guide/docs/env.html#nccl-net-gdr-level-formerly-nccl-ib-gdr-level).
+export NCCL_NET_GDR_LEVEL=PHB
+export NCCL_CROSS_NIC=1
+# These `FI` (libfabric) environment variables have been found to give the best
+# performance on the Alps network across a wide range of applications. Specific
+# applications may perform better with other values.
+export FI_CXI_DEFAULT_CQ_SIZE=131072
+export FI_CXI_DEFAULT_TX_SIZE=32768
+export FI_CXI_DISABLE_HOST_REGISTER=1
+export FI_CXI_RX_MATCH_MODE=software
+export FI_MR_CACHE_MONITOR=userfaultfd