Fix reporting on rerun-tests

* Echo rerun-rounds starting from 1, hide rebot stdout
* Zip log-files from previous test-rounds

Author: vsalomaki

.github/workflows/docker-build.yml

@@ -158,5 +158,5 @@ jobs:
         run: |
           podman run --user=2000:2000 --shm-size=1g \
             -v `pwd`/test/rerun.robot:/opt/robotframework/tests/rerun.robot:Z \
-            -e ROBOT_RERUN_FAILED=20 \
+            -e ROBOT_RERUN_FAILED=50 \
             $IMAGE_NAME

Dockerfile

@@ -18,6 +18,9 @@ ENV ROBOT_WORK_DIR /opt/robotframework/temp
 # Set the maximum number of rounds to rerun failed tests
 ENV ROBOT_RERUN_FAILED 0
 
+# Options to the rebot command when rerunning failed tests
+ENV ROBOT_RERUN_REBOT_OPTIONS ""
+
 # Setup X Window Virtual Framebuffer
 ENV SCREEN_COLOUR_DEPTH 24
 ENV SCREEN_HEIGHT 1080

README.md

@@ -14,6 +14,7 @@
     * [Testing emails](#testing-emails)
     * [Dealing with Datetimes and Timezones](#dealing-with-datetimes-and-timezones)
     * [Installing additional dependencies](#installing-additional-dependencies)
+    * [Rerunning tests](#rerunning-tests)
 * [Security consideration](#security-consideration)
 * [Continuous integration](#continuous-integration)
     * [Azure DevOps pipeline](#ci-azure-devops)

bin/run-tests-in-virtual-screen.sh

@@ -53,11 +53,15 @@ fi
 
 ROBOT_EXIT_CODE=$?
 
-if [[ ${ROBOT_EXIT_CODE} -gt 0 ]]
+if [ ${ROBOT_EXIT_CODE} -gt 0 ]
 then
     for ((i = 0 ; i < ${ROBOT_RERUN_FAILED} ; i++ ))
     do
-        echo "Rerunning failed tests, round ${i}..."
+        echo "Rerunning failed tests, round $((i+1))..."
+
+        # Store previous run into a tarball as screenshots etc from the previous run are removed by rebot
+        (cd $ROBOT_REPORTS_FINAL_DIR && tar czf report_run_${i}.tar.gz browser ./{*.html,*.xml,*.log,*.png})
+
         xvfb-run \
             --server-args="-screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_COLOUR_DEPTH} -ac" \
             robot \
@@ -68,11 +72,16 @@ then
             $ROBOT_TESTS_DIR
 
         ROBOT_EXIT_CODE=$?
-
+       
         rebot \
             --outputDir $ROBOT_REPORTS_FINAL_DIR \
-            --merge $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \
-            $ROBOT_REPORTS_FINAL_DIR/output.xml
+            --output $ROBOT_REPORTS_FINAL_DIR/output.xml \
+            --merge $ROBOT_REPORTS_FINAL_DIR/output.xml \
+            ${ROBOT_RERUN_REBOT_OPTIONS} \
+            $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \
+            > /dev/null
+        
+        rm $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml
 
         if [ ${ROBOT_EXIT_CODE} -eq 0 ]
         then

test/rerun.robot

@@ -1,7 +1,25 @@
 *** Test Cases ***
 
-Randomly Fail Test
-    ${rand}=    Evaluate    random.randint(0,3) 
-	IF    ${rand} > 0
+Roll Dice 1
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 2
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 3
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 4
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
         Fail
 	END