Bug: Fix log file handling in test-reruns (#539)

* Update Firefox to version 142.0

* Fix reporting on rerun-tests
* Echo rerun-rounds starting from 1, hide rebot stdout
* Zip log-files from previous test-rounds

* Rename ROBOT_RERUN_FAILED to ROBOT_RERUN_MAX_ROUNDS

Author: vsalomaki

.github/workflows/docker-build.yml

@@ -33,8 +33,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Build image
-        run:
-          podman build . --tag $IMAGE_NAME --file Dockerfile
+        run: podman build . --tag $IMAGE_NAME --file Dockerfile
       - name: Basic Test
         run: |
           podman run --shm-size=1g \
@@ -158,5 +157,5 @@ jobs:
         run: |
           podman run --user=2000:2000 --shm-size=1g \
             -v `pwd`/test/rerun.robot:/opt/robotframework/tests/rerun.robot:Z \
-            -e ROBOT_RERUN_FAILED=20 \
+            -e ROBOT_RERUN_MAX_ROUNDS=50 \
             $IMAGE_NAME

Dockerfile

@@ -16,7 +16,10 @@ ENV ROBOT_TESTS_DIR /opt/robotframework/tests
 ENV ROBOT_WORK_DIR /opt/robotframework/temp
 
 # Set the maximum number of rounds to rerun failed tests
-ENV ROBOT_RERUN_FAILED 0
+ENV ROBOT_RERUN_MAX_ROUNDS 0
+
+# Options to the rebot command when rerunning failed tests
+ENV ROBOT_RERUN_REBOT_OPTIONS ""
 
 # Setup X Window Virtual Framebuffer
 ENV SCREEN_COLOUR_DEPTH 24
@@ -44,7 +47,7 @@ ENV DATADRIVER_VERSION 1.11.2
 ENV DATETIMETZ_VERSION 1.0.6
 ENV MICROSOFT_EDGE_VERSION 139.0.3405.86
 ENV FAKER_VERSION 6.0.0
-ENV FIREFOX_VERSION 141.0
+ENV FIREFOX_VERSION 142.0
 ENV FTP_LIBRARY_VERSION 1.9
 ENV GECKO_DRIVER_VERSION v0.36.0
 ENV IMAP_LIBRARY_VERSION 0.4.11

README.md

@@ -14,6 +14,7 @@
     * [Testing emails](#testing-emails)
     * [Dealing with Datetimes and Timezones](#dealing-with-datetimes-and-timezones)
     * [Installing additional dependencies](#installing-additional-dependencies)
+    * [Rerunning tests](#rerunning-tests)
 * [Security consideration](#security-consideration)
 * [Continuous integration](#continuous-integration)
     * [Azure DevOps pipeline](#ci-azure-devops)
@@ -63,7 +64,7 @@ The versions used are:
 * [Robot Framework SeleniumLibrary](https://github.com/robotframework/SeleniumLibrary) 6.7.1
 * [Robot Framework SSHLibrary](https://github.com/robotframework/SSHLibrary) 3.8.0
 * [Axe Selenium Library](https://github.com/mozilla-services/axe-selenium-python) 2.1.6
-* Firefox 141.0
+* Firefox 142.0
 * [Chrome for Testing](https://googlechromelabs.github.io/chrome-for-testing/) 139.0
 * Microsoft Edge 139.0
 * [Amazon AWS CLI](https://pypi.org/project/awscli/) 1.42.11
@@ -201,15 +202,15 @@ rpa==1.50.0
 
 ### Rerunning tests
 
-Failing tests can be rerun by setting the environment variable `ROBOT_RERUN_FAILED` to a value above 0. All reruns of failed tests are executed without parallelization.
-The number in environment variable `ROBOT_RERUN_FAILED` dictates how many rerun-rounds are made at maximum. All rerun-rounds will only test what failed in the previous round.
+Failing tests can be rerun by setting the environment variable `ROBOT_RERUN_MAX_ROUNDS` to a value above 0. All reruns of failed tests are executed without parallelization.
+The number in environment variable `ROBOT_RERUN_MAX_ROUNDS` dictates how many rerun-rounds are made at maximum. All rerun-rounds will only test what failed in the previous round.
 The report files combine the results of all rounds, the last round providing the final result.
 
-The default value for `ROBOT_RERUN_FAILED` is 0, meaning that tests will not be executed again if they fail.
+The default value for `ROBOT_RERUN_MAX_ROUNDS` is 0, meaning that tests will not be executed again if they fail.
 
 ```sh
 docker run \
-    -e ROBOT_RERUN_FAILED=1 \
+    -e ROBOT_RERUN_MAX_ROUNDS=1 \
     ppodgorsek/robot-framework:latest
 ```
 

bin/run-tests-in-virtual-screen.sh

@@ -53,11 +53,15 @@ fi
 
 ROBOT_EXIT_CODE=$?
 
-if [[ ${ROBOT_EXIT_CODE} -gt 0 ]]
+if [ ${ROBOT_EXIT_CODE} -gt 0 ]
 then
-    for ((i = 0 ; i < ${ROBOT_RERUN_FAILED} ; i++ ))
+    for ((i = 0 ; i < ${ROBOT_RERUN_MAX_ROUNDS} ; i++ ))
     do
-        echo "Rerunning failed tests, round ${i}..."
+        echo "Rerunning failed tests, round $((i+1))..."
+
+        # Store previous run into a tarball as screenshots etc from the previous run are removed by rebot
+        (cd $ROBOT_REPORTS_FINAL_DIR && tar czf report_run_${i}.tar.gz browser *.html *.xml *.log *.png)
+
         xvfb-run \
             --server-args="-screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_COLOUR_DEPTH} -ac" \
             robot \
@@ -68,11 +72,16 @@ then
             $ROBOT_TESTS_DIR
 
         ROBOT_EXIT_CODE=$?
-
+       
         rebot \
             --outputDir $ROBOT_REPORTS_FINAL_DIR \
-            --merge $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \
-            $ROBOT_REPORTS_FINAL_DIR/output.xml
+            --output $ROBOT_REPORTS_FINAL_DIR/output.xml \
+            --merge $ROBOT_REPORTS_FINAL_DIR/output.xml \
+            ${ROBOT_RERUN_REBOT_OPTIONS} \
+            $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \
+            > /dev/null
+        
+        rm $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml
 
         if [ ${ROBOT_EXIT_CODE} -eq 0 ]
         then

test/rerun.robot

@@ -1,7 +1,25 @@
 *** Test Cases ***
 
-Randomly Fail Test
-    ${rand}=    Evaluate    random.randint(0,3) 
-	IF    ${rand} > 0
+Roll Dice 1
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 2
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 3
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 4
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
         Fail
 	END