fixing deepspeed on 64 gpus

Author: adityaranjan

scripts/out.png

@@ -17,14 +17,14 @@ export MPICH_GPU_SUPPORT_ENABLED=0
 export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${CRAY_MPICH_ROOTDIR}/gtl/lib"
 
 ## this enables the slingshot-11 plugin for RCCL (crucial for inter-node bw)
-# export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/lustre/orion/scratch/adityaranjan/csc547/my-venv/aws-ofi-rccl/build/lib"
+export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:/lustre/orion/scratch/adityaranjan/csc547/my-venv/aws-ofi-rccl/build/lib"
 # export NCCL_DEBUG=INFO
-# export FI_CXI_ATS=0
+export FI_CXI_ATS=0
 
 ## this improves cross node bandwidth for some cases
-# export NCCL_CROSS_NIC=1
+export NCCL_CROSS_NIC=1
 
-# export CUDA_DEVICE_MAX_CONNECTIONS=1
+export CUDA_DEVICE_MAX_CONNECTIONS=1
 
 export PYTHONPATH="${PYTHONPATH}:/ccs/home/adityaranjan/scratch/my-venv/improved-diffusion"