Fix retry mechanism in FV/Nv online tests

Changed way we call retry in shell

Relates-to: OLPEDGE-1144

Signed-off-by: Yaroslav Stefinko <[email protected]>

Author: ystefinko

scripts/linux/fv/gitlab-olp-cpp-sdk-functional-test.sh

@@ -33,32 +33,12 @@ do
         sleep 0.15
         set -e
 done
-result=0
+
 echo ">>> Functional Test ... >>>"
 source $FV_HOME/olp-cpp-sdk-functional-test.variables
 $REPO_HOME/build/tests/functional/olp-cpp-sdk-functional-tests \
     --gtest_output="xml:$REPO_HOME/reports/olp-functional-test-report.xml" \
     --gtest_filter="-ArcGisAuthenticationTest.SignInArcGis":"FacebookAuthenticationTest.SignInFacebook"
-result=$?
-echo "Last return code in $result"
-
-# Add retry to functional/online tests. Some online tests are flaky due to third party reason
-# Test failure should return code 1
-retry_count=0
-while [[ ${result} = 1 ]];
-do
-    retry_count=$((retry_count+1)) && echo "This is ${retry_count} time retry ..."
-    $REPO_HOME/build/tests/functional/olp-cpp-sdk-functional-tests \
-        --gtest_output="xml:$REPO_HOME/reports/olp-functional-test-report.xml" \
-        --gtest_filter="-ArcGisAuthenticationTest.SignInArcGis":"FacebookAuthenticationTest.SignInFacebook"
-    result=$?
-
-    # Stop after 3 retry
-    if [[ ${retry_count} = 3 ]]; then
-        break
-    fi
-done
-# End of retry part. This part can be removed anytime.
 
 # Kill local server
 kill -15 ${SERVER_PID}

scripts/linux/fv/gitlab_test_fv.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 #
 # Copyright (C) 2019 HERE Europe B.V.
 #
@@ -29,6 +29,7 @@ if [ -z $production_service_secret ] || [ -z $production_service_id ] ; then
 fi
 
 TEST_FAILURE=0
+RETRY_COUNT=0
 EXPECTED_REPORT_COUNT=6                         # expected that we generate 6 reports
 
 #for core dump backtrace
@@ -42,7 +43,35 @@ ${FV_HOME}/gitlab-olp-cpp-sdk-dataservice-read-test.sh 2>> errors.txt || TEST_FA
 ${FV_HOME}/gitlab-olp-cpp-sdk-dataservice-write-test.sh 2>> errors.txt || TEST_FAILURE=1
 
 # Run functional tests
-${FV_HOME}/gitlab-olp-cpp-sdk-functional-test.sh 2>> errors.txt || TEST_FAILURE=1
+# Add retry to functional/online tests. Some online tests are flaky due to third party reason
+# Test failure must return code 1 only. Other return code are not handled in retry.
+# Allowing tests failures (exit codes not 0) for online tests is done by set command below. Process is controlled by retry mechanism below.
+set +e
+while true
+do
+    # Stop after 3 retry
+    if [[ ${RETRY_COUNT} -eq 3 ]]; then
+        echo "Reach limit (${RETRY_COUNT}) of retries ..."
+        break
+    fi
+
+    RETRY_COUNT=$((RETRY_COUNT+1))
+    echo "This is ${RETRY_COUNT} time retry ..."
+
+    # Run functional tests
+    ${FV_HOME}/gitlab-olp-cpp-sdk-functional-test.sh 2>> errors.txt
+    if [[ $? -eq 1 ]]; then
+        TEST_FAILURE=1
+        continue
+    else
+        # Return to success
+        TEST_FAILURE=0
+        break
+    fi
+done
+set -e
+# End of retry part. This part can be removed anytime.
+
 
 # Run integration tests
 ${FV_HOME}/gitlab-olp-cpp-sdk-integration-test.sh 2>> errors.txt || TEST_FAILURE=1

scripts/linux/nv/gitlab_test_valgrind.sh

@@ -1,4 +1,4 @@
-#!/bin/bash -e
+#!/bin/bash -ex
 #
 # Copyright (C) 2019 HERE Europe B.V.
 #
@@ -85,20 +85,34 @@ do
     echo "-----> Finished ${test_group_name} - Result=${result}"
 
     # Add retry to functional/online tests. Some online tests are flaky due to third party reason
-    # Test failure should return code 1
-    retry_count=0
-    while [[ ${result} = 1 && ${test_group_name} == "functional" ]];
+    # Test failure must return code 1 only. Other return code are not handled in retry.
+    RETRY_COUNT=0
+    while true
     do
-        retry_count=$((retry_count+1)) && echo "This is ${retry_count} time retry ..."
-        echo "-----> Calling \"${test_command}\" for ${test_group_name} : "
-        eval "${test_command}"
-        result=$?
-        echo "-----> Finished ${test_group_name} - Result=${result}"
-
-        # Stop after 3 retry
-        if [[ ${retry_count} = 3 ]]; then
-            break
+        if [[ ${test_group_name} == "functional" ]]; then
+            # Stop after 3 retry
+            if [[ ${RETRY_COUNT} -eq 3 ]]; then
+                echo "Reach limit (${RETRY_COUNT}) of retries ..."
+                break
+            fi
+            RETRY_COUNT=$((RETRY_COUNT+1))
+            echo "This is ${RETRY_COUNT} time retry ..."
+
+            # Run functional tests
+            echo "-----> Calling \"${test_command}\" for ${test_group_name} : "
+            eval "${test_command}"
+            result=$?
+            echo "-----> Finished ${test_group_name} - Result=${result}"
+            if [[ ${result} -eq 1 ]]; then
+                TEST_FAILURE=1
+                continue
+            else
+                # Return to success
+                TEST_FAILURE=0
+                break
+            fi
         fi
+        break
     done
     # End of retry part. This part can be removed anytime.
 }