Test pt 26 (#5142)

Author: junpuf

test/dlc_tests/container_tests/bin/pytorch_tests/evaluateResults

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+set -e
+
+# setup benchmark repo
+git clone https://github.com/pytorch/benchmark.git
+cd benchmark
+PYTORCH_VERSION=$(python -c "import torch; print(torch.__version__)" 2>/dev/null || echo "pytorch not found")
+# Add git checkout for PyTorch 2.6
+if [[ "$PYTORCH_VERSION" == "2.6"* ]]; then
+    git checkout 4c7763b619b4eb1de88e57aee901eda92e270d7b
+fi
+pip install -r requirements.txt
+pip install numba

test/dlc_tests/container_tests/bin/pytorch_tests/installOSSPyTorch

@@ -2,25 +2,30 @@
 
 set -e
 
-source /test/bin/pytorch_tests/setupPyTorchBackendTest
+# check GPU status
+if nvidia-smi &> /dev/null; then
+   DEVICE="cuda"
+   NUM_GPUS=$(nvidia-smi -L | wc -l)
+else
+   DEVICE="cpu"
+   NUM_GPUS=0
+fi
+
+git config --global user.email "[email protected]"
+git config --global user.name "Your Name"
+
+if which conda >/dev/null 2>&1; then
+    conda init
+    source ~/.bashrc
+fi
+
+source /test/bin/pytorch_tests/installPyTorchBenchmarkRepository
 
 export PT_BACKEND=$1
 export USE_INDUCTOR=$2
 # assign the lower of 2 and gpu #, or 2 when no gpu
 export WORLD_SIZE=$(($NUM_GPUS == 0 ? 2 : ($NUM_GPUS < 2 ? $NUM_GPUS : 2)))
 
-# test aws pytorch
 OUT_PATH=$AWS_LOG_DIR/aws_res.csv timeout 360 python run.py BERT_pytorch -d $DEVICE -t train
 
-bash ${BIN_DIR}/pytorch_tests/installOSSPyTorch
-
-# test oss pytorch
-OUT_PATH=$OSS_LOG_DIR/oss_res.csv timeout 360 python run.py BERT_pytorch -d $DEVICE -t train
-
-# compare aws and oss results process by process (result metrics are aleady medians)
-for ((i=0; i<$WORLD_SIZE; i++))
-do
-  bash ${BIN_DIR}/pytorch_tests/evaluateResults "$AWS_LOG_DIR/aws_res.csv_$i" "$OSS_LOG_DIR/oss_res.csv_$i"
-done
-
 exit 0

test/dlc_tests/container_tests/bin/pytorch_tests/installPyTorchBenchmarkRepository

@@ -2,7 +2,21 @@
 
 set -e
 
-source /test/bin/pytorch_tests/setupPyTorchBackendTest
+export HOME_DIR=/test
+export LOG_DIR=${HOME_DIR}/logs
+export AWS_LOG_DIR=$LOG_DIR/aws_log
+
+# check GPU status
+if nvidia-smi &> /dev/null; then
+   DEVICE="cuda"
+   NUM_GPUS=$(nvidia-smi -L | wc -l)
+else
+   DEVICE="cpu"
+   NUM_GPUS=0
+fi
+
+source /test/bin/pytorch_tests/installPyTorchBenchmarkRepository
+
 export USE_INDUCTOR=$1
 echo testPyTorchNCCL: USE_INDUCTOR, $USE_INDUCTOR
 
@@ -19,38 +33,11 @@ python userbenchmark/ddp_experiments/run.py \
     --job_dir $AWS_LOG_DIR \
     --nccl-socket-ifname eth0
 
-
 # generate the result csv
 JOB_ID=$(ls $AWS_LOG_DIR | grep .out | head -n 1 | cut -d'_' -f 1)
 python userbenchmark/ddp_experiments/parse_ddp.py \
     --job_id $JOB_ID \
     --results_dir $AWS_LOG_DIR \
     --csv_out > $AWS_LOG_DIR/aws_res.csv
 echo testPyTorchNCCL: AWS-PyTorch test results && cat $AWS_LOG_DIR/aws_res.csv
-
-# get OSS perf data
-bash ${BIN_DIR}/pytorch_tests/installOSSPyTorch
-
-# run NCCL benchmarking with 2 GPUS and 1 local node (resnet will use torchvision)
-python userbenchmark/ddp_experiments/run.py \
-    --ngpus $WORLD_SIZE \
-    --distributed ddp \
-    --nodes 1 \
-    --cluster local \
-    --filter_models resnet50 \
-    --timeout 10 \
-    --job_dir $OSS_LOG_DIR \
-    --nccl-socket-ifname eth0
-
-# generate the result csv
-JOB_ID=$(ls $OSS_LOG_DIR | grep .out | head -n 1 | cut -d'_' -f 1)
-python userbenchmark/ddp_experiments/parse_ddp.py \
-    --job_id $JOB_ID \
-    --results_dir $OSS_LOG_DIR \
-    --csv_out > $OSS_LOG_DIR/oss_res.csv
-echo testPyTorchNCCL: OSS-PyTorch test results && cat $OSS_LOG_DIR/oss_res.csv
-
-# evaluate the performance
-bash ${BIN_DIR}/pytorch_tests/evaluateResults $AWS_LOG_DIR/aws_res.csv $OSS_LOG_DIR/oss_res.csv
-
-exit 0
+exit 0