DISTMYSQL-251: Orchestrator tests fail with PS 8.0.30

1. Implemented retry logic for system tests. Some tests seem to be not
stable, they pass when retried. Also improved test results reporting
(passed/unstable/failed)
2. After restore, some nodes may report non-zero replication lag. It has
no impact for tests, but caused check failure. Masked the lag in outpu.
3. Updated scripts to use PS 8.0.30 by default

Author: kamil-holubicki

docker/Dockerfile.system

@@ -27,7 +27,7 @@ RUN (cd /orchestrator/orchestrator-ci-env/bin/linux && ln -s /dbdeployer/dbdeplo
 
 # For dev purposes only, just to avoid downloading over and over via download-mysql script
 # RUN (mkdir /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded)
-# COPY docker/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded/ 
+# COPY docker/Percona-Server-8.0.30-22-Linux.x86_64.glibc2.17-minimal.tar.gz /orchestrator/orchestrator-ci-env/mysql-tarballs-downloaded/ 
 
 RUN (cd /orchestrator/orchestrator-ci-env && cp bin/linux/systemctl.py /usr/bin/systemctl)
 RUN (cd /orchestrator/orchestrator-ci-env && script/deploy-haproxy)

run/test-system.sh

@@ -7,7 +7,7 @@ export CI_ENV_REPO=
 export CI_ENV_BRANCH=
 
 # Configure test run parameters
-export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.26-16/binary/tarball/Percona-Server-8.0.26-16-Linux.x86_64.glibc2.12-minimal.tar.gz
+export TARBALL_URL=https://downloads.percona.com/downloads/Percona-Server-8.0/Percona-Server-8.0.30-22/binary/tarball/Percona-Server-8.0.30-22-Linux.x86_64.glibc2.17-minimal.tar.gz
 export RUN_TESTS=YES
 export ALLOW_TESTS_FAILURES=YES
 

tests/system/check_restore

@@ -1,3 +1,6 @@
 #!/bin/bash
 
-orchestrator-client -c topology-tabulated -alias ci | cut -d'|' -f 1,2,3
+# sed is to get rid of lag tag (Ns) part as it may fluctuate, but it does not affect tests.
+# "127.0.0.1:10112|1s|ok" -> "127.0.0.1:10112|LAG|ok"
+# It does not change "unknown", "detached" and other non-numeric values.
+orchestrator-client -c topology-tabulated -alias ci | sed -n "s/|[0-9]\+s|/|LAG|/p" | cut -d'|' -f 1,2,3

tests/system/expect_restore

@@ -1,4 +1,4 @@
-127.0.0.1:10111  |0s|ok
-+ 127.0.0.1:10112|0s|ok
-+ 127.0.0.1:10113|0s|ok
-+ 127.0.0.1:10114|0s|ok
+127.0.0.1:10111  |LAG|ok
++ 127.0.0.1:10112|LAG|ok
++ 127.0.0.1:10113|LAG|ok
++ 127.0.0.1:10114|LAG|ok

tests/system/test.sh

@@ -301,40 +301,84 @@ should_attempt_test() {
 }
 
 test_all() {
+  local passed_cnt=0
+  local unstable_cnt=0
+  local failed_cnt=0
+  local total_cnt=0
+  local expected_cnt=`find $tests_path -mindepth 1 -maxdepth 1 ! -path . -type d | wc -l`
+  
   local test_pattern="${1:-.}"
 
   echo "." > $tests_todo_file
   while [ -s $tests_todo_file ] ; do
     echo -n > $tests_todo_file
-
-    find $tests_path -mindepth 1 -maxdepth 1 ! -path . -type d | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern" | while read test_name ; do
+    while read test_name ; do
       if ! test_listed_as_attempted "$test_name" ; then
         echo "$test_name" >> $tests_todo_file
       fi
       if should_attempt_test "$test_name" "$test_pattern" ; then
-        test_single "$test_name"
-        if [ $? -eq 0 ] ; then
+        # try up to 3 times. Some tests seem to be not stable
+        
+        test_status="PASSED"
+        for i in {1..3}
+        do
+          if [ ${i} -ne 1 ] ; then
+            test_status="UNSTABLE"
+            echo "Retrying test after failure (${i}/3). Testname: $test_name"
+            # cleanup the state just in case
+            bash $tests_path/deploy-replication || return 1
+          fi
+          test_single "$test_name"
+          test_single_result=$?
+          if [ $test_single_result -eq 0 ] ; then
+            break
+          fi
+        done
+        if [ $test_single_result -eq 0 ] ; then
+          echo "Test finished. Testname: ${test_name} status: ${test_status}"
           echo "$test_name" >> $tests_successful_file
         else
           echo "$test_name" >> $tests_failed_file
+          test_status="FAILED"
+          echo "Test finished. Testname: ${test_name} status: ${test_status}"
           if [ "$ALLOW_TESTS_FAILURES" != "YES" ] ; then
             echo "Tests failures not allowed. Exiting."
             exit 1
           else
             echo "Tests failures allowed. Continuing."
+            # cleanup the state just in case
+            bash $tests_path/deploy-replication || return 1
           fi
         fi
+        if [ "${test_status}" == "PASSED" ] ; then
+          ((passed_cnt++))
+        elif [ "${test_status}" == "UNSTABLE" ] ; then
+          ((unstable_cnt++))
+        elif [ "${test_status}" == "FAILED" ] ; then
+          ((failed_cnt++))
+        else
+          echo "Unexpected test_status: ${test_status}"
+        fi
+        ((total_cnt++))    
       else
         : # echo "# should not attempt $test_name"
       fi
-    done || return 1
+    done < <(find $tests_path -mindepth 1 -maxdepth 1 ! -path . -type d | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern") || return 1
   done
   find $tests_path -mindepth 1 -maxdepth 1 ! -path . -type d | xargs ls -td1 | cut -d "/" -f 4 | egrep "$test_pattern" | while read test_name ; do
     if ! test_listed_as_attempted "$test_name" ; then
       echo "# ERROR: tests completed by $test_name seems to have been skipped"
       exit 1
     fi
   done || exit 1
+  
+  echo "Test results:"
+  echo "PASSED:   ${passed_cnt}/${total_cnt}/${expected_cnt}"
+  echo "UNSTABLE: ${unstable_cnt}/${total_cnt}/${expected_cnt}"
+  echo "FAILED:   ${failed_cnt}/${total_cnt}/${expected_cnt}"
+  if [ ${total_cnt} -ne ${expected_cnt} ] ; then
+    echo "WARNING! Some tests were skipped. Expected: ${expected_cnt}, executed: ${total_cnt}"
+  fi
 }
 
 main() {