Merge branch '2.10.2.1' into 2.10.2.1_ds_monitoring

Author: shethdhvani

playbooks/roles/nvidia_peermem/tasks/common.yml

@@ -3,23 +3,27 @@
   shell:
     cmd: "curl -sH \"Authorization: Bearer Oracle\" -L http://169.254.169.254/opc/v2/instance/ | jq .shape | grep GPU"
     warn: false
-  register: shape
+  register: shape_gpu
   failed_when: false
 
-
 - name: Check if nvidia drivers are installed
   shell: cat /sys/module/nvidia/version | wc -l
   register: nvidia
-  when: shape.stdout != ""
-
+  when: shape_gpu.stdout != ""
 
 - name: Check if nvidia_peermem module is loaded
   shell: lsmod | grep nvidia_peermem | wc -l
   register: result
-  when: shape.stdout != "" and nvidia.stdout == '1'
+  when: shape_gpu.stdout != "" and nvidia.stdout == '1'
 
+- name: Check ofed version
+  shell:
+    cmd: |
+      /usr/bin/ofed_info |grep MLNX_OFED_LINUX|grep -v rpm|awk -F "(" '{print $2}'|cut -c 6-|awk -F "-" '{print $1}'
+  register: ofed_version_local
+  when: shape_gpu.stdout != "" and nvidia.stdout == '1'
 
 - name: Load nvidia_peermem module
   become: true
   shell: modprobe nvidia_peermem
-  when: shape.stdout != "" and nvidia.stdout == '1' and result.stdout != '3'
+  when: shape_gpu.stdout != "" and nvidia.stdout == '1' and result.stdout != '3' and ofed_version_local.stdout|int >= '5.1'

samples/gpu/nccl_run_allreduce.sh

@@ -68,6 +68,7 @@ fi
   # final version
   mpirun --mca pml ucx \
   --bind-to numa \
+  --mca coll ^hcoll \
   -x NCCL_DEBUG=WARN \
   -x NCCL_IB_SL=0 \
   -x NCCL_IB_TC=41 \

samples/gpu/nccl_run_alltoall.sh

@@ -73,8 +73,10 @@ fi
 
   # Use  -x NCCL_MAX_P2P_NCHANNELS=16 until NCCL 2.12 release which has a fix to allow NCCL_MAX_P2P_NCHANNELS=32 for nodes with 16 RDMA NICss
   # final version
+  # you need --mca coll ^hcoll when using an image that has OFED 5.4 or newer
   mpirun --mca pml ucx \
   --bind-to numa \
+  --mca coll ^hcoll \
   -x NCCL_MAX_P2P_NCHANNELS=16 \
   -x NCCL_DEBUG=WARN \
   -x NCCL_IB_SL=0 \

samples/gpu/qfabv1_nccl_run_allreduce.sbatch

@@ -53,8 +53,10 @@ then
   var_NCCL_IB_HCA="=mlx5_0,mlx5_2,mlx5_6,mlx5_8,mlx5_10,mlx5_12,mlx5_14,mlx5_16"
 fi
 
+# you need --mca coll ^hcoll when using an image that has OFED 5.4 or newer
   mpirun --mca pml ucx \
   --bind-to numa \
+  --mca coll ^hcoll \ 
   -x NCCL_DEBUG=WARN \
   -x NCCL_IB_SL=0 \
   -x NCCL_IB_TC=41 \

samples/gpu/qfabv1_nccl_run_allreduce.sh

@@ -66,9 +66,11 @@ then
   var_NCCL_IB_HCA="=mlx5_0,mlx5_2,mlx5_6,mlx5_8,mlx5_10,mlx5_12,mlx5_14,mlx5_16"
 fi
 
+# you need --mca coll ^hcoll when using an image that has OFED 5.4 or newer
   # final version
   mpirun --mca pml ucx \
   --bind-to numa \
+  --mca coll ^hcoll \ 
   -x NCCL_DEBUG=WARN \
   -x NCCL_IB_SL=0 \
   -x NCCL_IB_TC=41 \

samples/gpu/qfabv1_nccl_run_alltoall.sh

@@ -78,8 +78,10 @@ fi
   # Use NCCL_IB_QPS_PER_CONNECTION=4 for QFAB1.0, should get around 15GB/s NCCL Bus BW. 
   # Use  -x NCCL_MAX_P2P_NCHANNELS=16 until NCCL 2.12 release which has a fix to allow NCCL_MAX_P2P_NCHANNELS=32 for nodes with 16 RDMA NICss
   # final version
+  # you need --mca coll ^hcoll when using an image that has OFED 5.4 or newer
   mpirun --mca pml ucx \
   --bind-to numa \
+  --mca coll ^hcoll \
   -x NCCL_MAX_P2P_NCHANNELS=16 \
   -x NCCL_DEBUG=WARN \
   -x NCCL_IB_SL=0 \