Chnage H100 Parameters for Single Subnet and NCCL 2.20.3

Author: arnaudfroidmont

playbooks/roles/nccl-conf/files/h100

@@ -1,5 +1,4 @@
-NCCL_CROSS_NIC=0
-NCCL_SOCKET_NTHREADS=16
+NCCL_CROSS_NIC=1
 NCCL_DEBUG=WARN
 NCCL_CUMEM_ENABLE=0
 NCCL_IB_SPLIT_DATA_ON_QPS=0
@@ -8,8 +7,8 @@ NCCL_IB_GID_INDEX=3
 NCCL_IB_TC=41
 NCCL_IB_SL=0
 NCCL_IB_TIMEOUT=22
+NCCL_BUFFSIZE=16777216
 NCCL_NET_PLUGIN=none
 NCCL_SOCKET_IFNAME=eth0
 NCCL_IGNORE_CPU_AFFINITY=1
-NCCL_IB_HCA==mlx5_0,mlx5_1,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7,mlx5_8,mlx5_9,mlx5_10,mlx5_12,mlx5_13,mlx5_14,mlx5_15,mlx5_16,mlx5_17
-NCCL_TOPO_FILE=/nfs/cluster/H100-topology.xml
+NCCL_IB_HCA==mlx5_0,mlx5_1,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7,mlx5_8,mlx5_9,mlx5_10,mlx5_12,mlx5_13,mlx5_14,mlx5_15,mlx5_16,mlx5_17

playbooks/roles/nccl-conf/files/libnccl-ocituner.so.1.0.1_OL

@@ -38,7 +38,7 @@
 - name: copy libnccl-ocituner for OL
   become: true
   get_url: 
-    url: wget https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/m1Gdcbiguqst6n_aVwRZIFpRZxUG-wGMvqWS5QJeJbIvNZnqTTA3N1_DDRuYpvJx/n/hpc/b/source/o/tuner/libnccl-ocituner.so.1.0.1-OL
+    url: https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/m1Gdcbiguqst6n_aVwRZIFpRZxUG-wGMvqWS5QJeJbIvNZnqTTA3N1_DDRuYpvJx/n/hpc/b/source/o/tuner/libnccl-ocituner.so.1.0.1-OL
     dest: /home/opc/libnccl-ocituner.so.1.0.1
     owner: opc
     group: privilege
@@ -48,7 +48,7 @@
 - name: copy libnccl-ocituner for Ubuntu
   become: true
   get_url: 
-    url: wget https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/m1Gdcbiguqst6n_aVwRZIFpRZxUG-wGMvqWS5QJeJbIvNZnqTTA3N1_DDRuYpvJx/n/hpc/b/source/o/tuner/libnccl-ocituner.so.1.0.1-ubuntu
+    url: https://objectstorage.eu-frankfurt-1.oraclecloud.com/p/m1Gdcbiguqst6n_aVwRZIFpRZxUG-wGMvqWS5QJeJbIvNZnqTTA3N1_DDRuYpvJx/n/hpc/b/source/o/tuner/libnccl-ocituner.so.1.0.1-ubuntu
     dest: /home/ubuntu/libnccl-ocituner.so.1.0.1
     owner: ubuntu
     group: privilege

playbooks/roles/nccl-conf/files/libnccl-ocituner.so.1.0.1_ubuntu

@@ -43,8 +43,7 @@ fi
   --bind-to numa \
   -npernode 8 \
   --mca coll ^hcoll \
-  -x NCCL_CROSS_NIC=0 \
-  -x NCCL_SOCKET_NTHREADS=16 \
+  -x NCCL_CROSS_NIC=1 \
   -x NCCL_DEBUG=WARN \
   -x NCCL_CUMEM_ENABLE=0 \
   -x NCCL_IB_SPLIT_DATA_ON_QPS=0 \
@@ -60,8 +59,11 @@ fi
   -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \
   -x RX_QUEUE_LEN=8192 \
   -x IB_RX_QUEUE_LEN=8192 \
+  -x NCCL_BUFFSIZE=16777216 \
   -x NCCL_SOCKET_IFNAME=eth0 \
   -x NCCL_IGNORE_CPU_AFFINITY=1 \
   -x NCCL_IB_HCA="${var_NCCL_IB_HCA}" \
-  -x NCCL_TOPO_FILE=~/H100-topology.xml \
-  --np $((SLURM_NNODES*SLURM_NTASKS_PER_NODE)) --hostfile $MACHINEFILE  /opt/oci-hpc/nccl-test/build/all_reduce_perf -b 1G -e 16G -f 2 -g 1
+  --np $((SLURM_NNODES*SLURM_NTASKS_PER_NODE)) --hostfile $MACHINEFILE  /opt/oci-hpc/nccl-test/build/all_reduce_perf -b 1G -e 16G -f 2 -g 1
+
+  # If NCCL version is lower than 2.20.3, it is recommended to use the topology filefor optimal performances 
+  # -x NCCL_TOPO_FILE=~/H100-topology.xml \

playbooks/roles/nccl-conf/tasks/main.yml

@@ -57,8 +57,7 @@ do
   --bind-to numa \
   -npernode 8 \
   --mca coll ^hcoll \
-  -x NCCL_CROSS_NIC=0 \
-  -x NCCL_SOCKET_NTHREADS=16 \
+  -x NCCL_CROSS_NIC=1 \
   -x NCCL_DEBUG=WARN \
   -x NCCL_CUMEM_ENABLE=0 \
   -x NCCL_IB_SPLIT_DATA_ON_QPS=0 \
@@ -74,6 +73,7 @@ do
   -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \
   -x RX_QUEUE_LEN=8192 \
   -x IB_RX_QUEUE_LEN=8192 \
+  -x NCCL_BUFFSIZE=16777216 \
   -x NCCL_SOCKET_IFNAME=eth0 \
   -x NCCL_IGNORE_CPU_AFFINITY=1 \
   -x NCCL_IB_HCA="${var_NCCL_IB_HCA}" \
@@ -82,5 +82,8 @@ do
 
   tail -n 32 $logfile
 
+done
 
-done
+
+  # If NCCL version is lower than 2.20.3, it is recommended to use the topology filefor optimal performances 
+  # -x NCCL_TOPO_FILE=~/H100-topology.xml \

samples/gpu/nccl_run_allreduce_H100.sbatch

@@ -21,9 +21,11 @@ cat $MACHINEFILE
 
 source /etc/os-release
 if [ $ID == "ol" ] || [ $ID == "centos" ] ; then
-    python3 /home/opc/node_ordering_by_rack.py --input_file $MACHINEFILE > /dev/null
+    python3 /home/opc/node_ordering_by_rack.py --input_file $hostfile > /dev/null
+    homedirectory=/home/opc
 elif [ $ID == "debian" ] || [ $ID == "ubuntu" ] ; then
-    python3 /home/ubuntu/node_ordering_by_rack.py --input_file $MACHINEFILE > /dev/null
+    python3 /home/ubuntu/node_ordering_by_rack.py --input_file $hostfile > /dev/null
+    homedirectory=/home/ubuntu
 fi