Merge branch '2.10.4' of https://github.com/oci-hpc/oci-hpc-clusternetwork-dev into 2.10.4

Author: arnaudfroidmont

samples/gpu/nccl_run_allreduce_H100.sbatch

@@ -7,7 +7,7 @@
 export PMI_DEBUG=1
 
 
-cd /nfs/scratch
+cd /nfs/cluster
 mkdir $SLURM_JOB_ID
 cd $SLURM_JOB_ID
 
@@ -30,8 +30,6 @@ if [[ "$mpivars_path" == "" ]]; then
 
 source $mpivars_path
 
-export NCCL_DEBUG=WARN
-
 shape=`curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/ | jq .shape`
 if [ $shape == \"BM.GPU.H100.8\" ]
 then
@@ -43,17 +41,28 @@ fi
 
   mpirun --mca pml ucx \
   --bind-to numa \
+  -npernode 8 \
   --mca coll ^hcoll \
+  -x NCCL_CROSS_NIC=0 \
+  -x NCCL_SOCKET_NTHREADS=16 \
   -x NCCL_DEBUG=WARN \
-  -x NCCL_IB_SL=0 \
-  -x NCCL_IB_TC=41 \
+  -x NCCL_CUMEM_ENABLE=0 \
+  -x NCCL_IB_SPLIT_DATA_ON_QPS=0 \
   -x NCCL_IB_QPS_PER_CONNECTION=16 \
-  -x UCX_TLS=tcp \
-  -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \
+  -x NCCL_IB_GID_INDEX=3 \
+  -x NCCL_IB_TC=41 \
+  -x NCCL_IB_SL=0 \
+  -x NCCL_IB_TIMEOUT=22 \
+  -x NCCL_NET_PLUGIN=none \
   -x HCOLL_ENABLE_MCAST_ALL=0 \
   -x coll_hcoll_enable=0 \
-  -x NCCL_IB_GID_INDEX=3 \
-  -x NCCL_ALGO=Auto \
+  -x UCX_TLS=tcp \
+  -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \
+  -x RX_QUEUE_LEN=8192 \
+  -x IB_RX_QUEUE_LEN=8192 \
+  -x NCCL_SOCKET_IFNAME=eth0 \
+  -x NCCL_ALGO=auto \
+  -x NCCL_IGNORE_CPU_AFFINITY=1 \
   -x NCCL_IB_HCA="${var_NCCL_IB_HCA}" \
   -x NCCL_TOPO_FILE=~/H100-topology.xml \
-  --np $((SLURM_NNODES*SLURM_NTASKS_PER_NODE))  --hostfile $MACHINEFILE  /opt/oci-hpc/nccl-test/build/all_reduce_perf -b1G -e10G -i$((1024*1024*1024*9)) -n 100
+  --np $((SLURM_NNODES*SLURM_NTASKS_PER_NODE)) --hostfile $MACHINEFILE  /opt/oci-hpc/nccl-test/build/all_reduce_perf -b1G -e10G -i$((1024*1024*1024*9)) -n 100

samples/gpu/nccl_run_allreduce_H100.sh

@@ -0,0 +1,87 @@
+#!/bin/bash
+set -e
+
+# number of times to run the nccl test to stress the GPUs and RDMA network. This is different from -n iterations parameter of nccl allreduce which is set below using $iter
+max=$1
+
+# This assume, the hostfile  passed is already ordered based on their rackId
+if [ -n "$2" ]; then
+  hostfile=$2
+else
+  hostfile="/etc/opt/oci-hpc/hostfile.tcp"
+fi
+
+echo INPUTFILE
+cat $hostfile
+
+# The number of GPUs to use for the test.  Has to be multiplier of 8.  If not passed, all GPUs will be used.
+if [ -n "$3" ]; then
+  np=$3
+else
+  np=$((`less $hostfile | wc -l` * 8 ))
+fi
+
+logfile="nccl_run_allreduce.sh.log"
+
+for x in $(seq 1 1 $max)
+do
+
+  echo $x
+  echo $x >> $logfile
+  date >> $logfile
+
+  hostfile=$hostfile; np=$np ; iter=20;
+
+  mpivars_path=`ls /usr/mpi/gcc/openmpi-*/bin/mpivars.sh`
+
+  if [[ "$mpivars_path" == "" ]]; then
+      mpivars_path=`ls /opt/openmpi-*/bin/mpivars.sh`
+  fi
+
+  if [[ "$mpivars_path" == "" ]]; then
+      echo "Could not find MPIPATH"; exit; fi
+
+  source $mpivars_path
+
+  first_node=`head $hostfile -n 1`
+  shape=`ssh $first_node 'curl -sH "Authorization: Bearer Oracle" -L http://169.254.169.254/opc/v2/instance/' | jq .shape`
+  if [ $shape == \"BM.GPU.H100.8\" ]
+  then
+    var_UCX_NET_DEVICES=eth0
+    var_NCCL_IB_HCA="=mlx5_0,mlx5_1,mlx5_3,mlx5_4,mlx5_5,mlx5_6,mlx5_7,mlx5_8,mlx5_9,mlx5_10,mlx5_12,mlx5_13,mlx5_14,mlx5_15,mlx5_16,mlx5_17"
+  else
+    echo "Use the appropriate nccl test run script for non H100 nodes"
+  fi
+
+  mpirun --mca pml ucx \
+  --bind-to numa \
+  -npernode 8 \
+  --mca coll ^hcoll \
+  -x NCCL_CROSS_NIC=0 \
+  -x NCCL_SOCKET_NTHREADS=16 \
+  -x NCCL_DEBUG=WARN \
+  -x NCCL_CUMEM_ENABLE=0 \
+  -x NCCL_IB_SPLIT_DATA_ON_QPS=0 \
+  -x NCCL_IB_QPS_PER_CONNECTION=16 \
+  -x NCCL_IB_GID_INDEX=3 \
+  -x NCCL_IB_TC=41 \
+  -x NCCL_IB_SL=0 \
+  -x NCCL_IB_TIMEOUT=22 \
+  -x NCCL_NET_PLUGIN=none \
+  -x HCOLL_ENABLE_MCAST_ALL=0 \
+  -x coll_hcoll_enable=0 \
+  -x UCX_TLS=tcp \
+  -x UCX_NET_DEVICES=${var_UCX_NET_DEVICES} \
+  -x RX_QUEUE_LEN=8192 \
+  -x IB_RX_QUEUE_LEN=8192 \
+  -x NCCL_SOCKET_IFNAME=eth0 \
+  -x NCCL_ALGO=auto \
+  -x NCCL_IGNORE_CPU_AFFINITY=1 \
+  -x NCCL_IB_HCA="${var_NCCL_IB_HCA}" \
+  -x NCCL_TOPO_FILE=~/H100-topology.xml \
+  --np $np --hostfile $hostfile  /opt/oci-hpc/nccl-test/build/all_reduce_perf -b1G -e10G -i$((1024*1024*1024*9)) -n 100 >>  $logfile
+
+  tail -n 32 $logfile
+
+
+done