fix: use standalone tests' exit code

Author: Borda

tests/run_standalone_tests.sh

@@ -26,72 +26,85 @@ export PL_RUN_STANDALONE_TESTS=1
 defaults=" -m coverage run --source ${source} --append -m pytest --no-header -v -s --timeout 120 "
 echo "Using defaults: ${defaults}"
 
-# get the testing location as the first argument
-test_path=$1
-printf "source path: $test_path\n"
+# get the list of parametrizations. we need to call them separately. the last two lines are removed.
+# note: if there's a syntax error, this will fail with some garbled output
+python3 -um pytest -q --collect-only --pythonwarnings ignore 2>&1 > $COLLECTED_TESTS_FILE
+# early terminate if collection failed (e.g. syntax error)
+if [[ $? != 0 ]]; then
+  cat $COLLECTED_TESTS_FILE
+  exit 1
+fi
 
-# collect all tests with parametrization based filtering with PL_RUN_STANDALONE_TESTS
-standalone_tests=$(python3 -m pytest $test_path -q --collect-only --pythonwarnings ignore)
-printf "Collected tests: \n $standalone_tests\n"
-# match only lines with tests
-parametrizations=$(perl -nle 'print $& while m{\S+::test_\S+}g' <<< "$standalone_tests")
-# convert the list to be array
-parametrizations_arr=($parametrizations)
-report=''
+# removes the last line of the file
+sed -i '$d' $COLLECTED_TESTS_FILE
 
-rm -f standalone_test_output.txt  # in case it exists, remove it
-rm -f testnames.txt
+# Get test list and run each test individually
+tests=($(grep -oP '\S+::test_\S+' "$COLLECTED_TESTS_FILE"))
+test_count=${#tests[@]}
+# present the collected tests
+printf "collected $test_count tests:\n-------------------\n"
+echo $(IFS='\n'; echo "${tests[@]}")
+printf "\n===================\n"
 
-function show_batched_output {
-  if [ -f standalone_test_output.txt ]; then  # if exists
-    cat standalone_test_output.txt
-    # heuristic: stop if there's mentions of errors. this can prevent false negatives when only some of the ranks fail
-    if perl -nle 'print if /error|(?<!(?-i)on_)exception|traceback|(?<!(?-i)x)failed/i' standalone_test_output.txt | grep -qv -f testnames.txt; then
-      echo "Potential error! Stopping."
-      perl -nle 'print if /error|(?<!(?-i)on_)exception|traceback|(?<!(?-i)x)failed/i' standalone_test_output.txt
-      rm standalone_test_output.txt
-      exit 1
-    fi
-    rm standalone_test_output.txt
-  fi
-}
-trap show_batched_output EXIT  # show the output on exit
-
-# remove the "tests/tests_pytorch/" path suffixes
-path_prefix=$(basename "$(dirname "$(pwd)")")/$(basename "$(pwd)")"/"  # https://stackoverflow.com/a/8223345
+# if test count is one print warning
+if [[ $test_count -eq 1 ]]; then
+  printf "WARNING: only one test found!\n"
+elif [ $test_count -eq 0 ]; then
+  printf "ERROR: no tests found!\n"
+  exit 1
+fi
 
-for i in "${!parametrizations_arr[@]}"; do
-  parametrization=${parametrizations_arr[$i]//$path_prefix/}
-  prefix="$((i+1))/${#parametrizations_arr[@]}"
+# clear all the collected reports
+rm -f parallel_test_output-*.txt  # in case it exists, remove it
 
-  echo "$prefix: Running $parametrization"
-  echo $parametrization | sed 's/\[[^][]*\]//g' >> testnames.txt
 
-  # fix the port to avoid race condition when batched distributed tests select the port randomly
-  export MASTER_PORT=$((29500 + $i % $test_batch_size))
+status=0 # reset the script status
+report="" # final report
+pids=() # array of PID for running tests
+test_ids=() # array of indexes of running tests
+printf "Running $test_count tests in batches of $test_batch_size\n"
+for i in "${!tests[@]}"; do
+  test=${tests[$i]}
+  printf "Running test $((i+1))/$test_count: $test\n"
 
   # execute the test in the background
-  # redirect to a log file that buffers test output. since the tests will run in the background, we cannot let them
-  # output to std{out,err} because the outputs would be garbled together
-  python3 ${defaults} "$parametrization" &>> standalone_test_output.txt &
-  # save the PID in an array
-  pids[${i}]=$!
-  # add row to the final report
-  report+="Ran\t$parametrization\n"
+  # redirect to a log file that buffers test output. since the tests will run in the background,
+  # we cannot let them output to std{out,err} because the outputs would be garbled together
+  python3 ${defaults} "$test" 2>&1 > "standalone_test_output-$i.txt" &
+  test_ids+=($i) # save the test's id in an array with running tests
+  pids+=($!) # save the PID in an array with running tests
 
-  if ((($i + 1) % $test_batch_size == 0)); then
+  # if we reached the batch size, wait for all tests to finish
+  if (( (($i + 1) % $test_batch_size == 0) || $i == $test_count-1 )); then
+    printf "Waiting for batch to finish: $(IFS=' '; echo "${pids[@]}")\n"
     # wait for running tests
-    for pid in ${pids[*]}; do wait $pid; done
-    unset pids  # empty the array
-    show_batched_output
+    for j in "${!test_ids[@]}"; do
+      i=${test_ids[$j]} # restore the global test's id
+      pid=${pids[$j]} # restore the particular PID
+      test=${tests[$i]} # restore the test name
+      printf "Waiting for $tests >> standalone_test_output-$i.txt (PID: $pid)\n"
+      wait -n $pid
+      # get the exit status of the test
+      test_status=$?
+      # add row to the final report
+      report+="Ran\t$test\t>> exit:$test_status\n"
+      if [[ $test_status != 0 ]]; then
+        # show the output of the failed test
+        cat "standalone_test_output-$i.txt"
+        # Process exited with a non-zero exit status
+        status=$test_status
+      fi
+    done
+    test_ids=()  # reset the test's id array
+    pids=()  # reset the PID array
   fi
 done
-# wait for leftover tests
-for pid in ${pids[*]}; do wait $pid; done
-show_batched_output
 
 # echo test report
 printf '=%.s' {1..80}
 printf "\n$report"
 printf '=%.s' {1..80}
 printf '\n'
+
+# exit with the worse test result
+exit $status