[integ-tests] Upgrade NCCL versions

The new NCCL version has some performance improvement on Blackwell.
See NCCL release note: https://docs.nvidia.com/deeplearning/nccl/release-notes/rel_2-28-3.html#rel_2-28-3

This upgrade makes NCCL performance on two p6-b200 15% better

Author: hanwen-cluster

tests/integration-tests/tests/common/data/nccl/init_nccl_benchmarks.sh

@@ -5,8 +5,8 @@ set -xe
 rm -rf /shared/${1}
 
 module load ${1}
-NCCL_BENCHMARKS_VERSION='2.16.7'
-NCCL_VERSION='2.27.7-1'
+NCCL_BENCHMARKS_VERSION='2.17.1'
+NCCL_VERSION='2.28.3-1'
 MPI_HOME=$(which mpirun | awk -F '/bin' '{print $1}')
 NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80 -gencode=arch=compute_90,code=sm_90 -gencode=arch=compute_90,code=compute_90" # Arch for NVIDIA A100 and H100, ref https://docs.nvidia.com/cuda/ada-compatibility-guide/index.html
 

tests/integration-tests/tests/common/data/nccl/nccl_tests_submit_openmpi.sh

@@ -3,8 +3,8 @@
 #SBATCH --exclusive
 
 module load openmpi
-NCCL_VERSION='2.27.7-1'
-NCCL_BENCHMARKS_VERSION='2.16.7'
+NCCL_VERSION='2.28.3-1'
+NCCL_BENCHMARKS_VERSION='2.17.1'
 
 . /etc/os-release
 if [[ $ID==rhel || $ID==rocky ]]; then