Correct reporting after rerun tests

* Echo rerun-rounds starting from 1, hide rebot stdout

Author: vsalomaki

.github/workflows/docker-build.yml

@@ -33,8 +33,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
       - name: Build image
-        run:
-          podman build . --tag $IMAGE_NAME --file Dockerfile
+        run: podman build . --tag $IMAGE_NAME --file Dockerfile
       - name: Basic Test
         run: |
           podman run --shm-size=1g \
@@ -158,5 +157,5 @@ jobs:
         run: |
           podman run --user=2000:2000 --shm-size=1g \
             -v `pwd`/test/rerun.robot:/opt/robotframework/tests/rerun.robot:Z \
-            -e ROBOT_RERUN_FAILED=20 \
+            -e ROBOT_RERUN_FAILED=50 \
             $IMAGE_NAME

Dockerfile

@@ -18,6 +18,9 @@ ENV ROBOT_WORK_DIR /opt/robotframework/temp
 # Set the maximum number of rounds to rerun failed tests
 ENV ROBOT_RERUN_FAILED 0
 
+# Options to the rebot command when rerunning failed tests
+ENV ROBOT_RERUN_REBOT_OPTIONS ""
+
 # Setup X Window Virtual Framebuffer
 ENV SCREEN_COLOUR_DEPTH 24
 ENV SCREEN_HEIGHT 1080
@@ -61,18 +64,18 @@ ENV AWS_UPLOAD_TO_S3 false
 # Install system dependencies
 RUN dnf upgrade -y --refresh \
   && dnf install -y \
-    dbus-glib \
-    dnf-plugins-core \
-    firefox-${FIREFOX_VERSION}* \
-    gcc \
-    gcc-c++ \
-    nodejs \
-    npm \
-    python3-pip \
-    python3-pyyaml \
-    tzdata \
-    wget \
-    xorg-x11-server-Xvfb-${XVFB_VERSION}* \
+  dbus-glib \
+  dnf-plugins-core \
+  firefox-${FIREFOX_VERSION}* \
+  gcc \
+  gcc-c++ \
+  nodejs \
+  npm \
+  python3-pip \
+  python3-pyyaml \
+  tzdata \
+  wget \
+  xorg-x11-server-Xvfb-${XVFB_VERSION}* \
   && dnf clean all
 
 # Install Chrome for Testing
@@ -112,15 +115,15 @@ RUN wget -q "https://github.com/mozilla/geckodriver/releases/download/$GECKO_DRI
 RUN rpm --import https://packages.microsoft.com/keys/microsoft.asc \
   && dnf config-manager addrepo --from-repofile=https://packages.microsoft.com/yumrepos/edge/config.repo \
   && dnf install -y \
-    microsoft-edge-stable-${MICROSOFT_EDGE_VERSION} \
-    zip \
+  microsoft-edge-stable-${MICROSOFT_EDGE_VERSION} \
+  zip \
   && wget -q "https://msedgedriver.microsoft.com/${MICROSOFT_EDGE_VERSION}/edgedriver_linux64.zip" \
   && unzip edgedriver_linux64.zip -d edge \
   && mv edge/msedgedriver /opt/robotframework/drivers/msedgedriver \
   && rm -Rf edgedriver_linux64.zip edge/ \
   # IMPORTANT: don't remove the wget package because it's a dependency of Microsoft Edge
   && dnf remove -y \
-    zip \
+  zip \
   && dnf clean all
 
 ENV PATH=/opt/microsoft/msedge:$PATH

README.md

@@ -14,6 +14,7 @@
     * [Testing emails](#testing-emails)
     * [Dealing with Datetimes and Timezones](#dealing-with-datetimes-and-timezones)
     * [Installing additional dependencies](#installing-additional-dependencies)
+    * [Rerunning tests](#rerunning-tests)
 * [Security consideration](#security-consideration)
 * [Continuous integration](#continuous-integration)
     * [Azure DevOps pipeline](#ci-azure-devops)

bin/run-tests-in-virtual-screen.sh

@@ -53,11 +53,15 @@ fi
 
 ROBOT_EXIT_CODE=$?
 
-if [[ ${ROBOT_EXIT_CODE} -gt 0 ]]
+if [ ${ROBOT_EXIT_CODE} -gt 0 ]
 then
     for ((i = 0 ; i < ${ROBOT_RERUN_FAILED} ; i++ ))
     do
-        echo "Rerunning failed tests, round ${i}..."
+        echo "Rerunning failed tests, round $((i+1))..."
+
+        # Store previous run into a tarball as screenshots etc from the previous run are removed by rebot
+        (cd $ROBOT_REPORTS_FINAL_DIR && tar czf report_run_${i}.tar.gz browser ./{*.html,*.xml,*.log,*.png})
+
         xvfb-run \
             --server-args="-screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_COLOUR_DEPTH} -ac" \
             robot \
@@ -68,11 +72,16 @@ then
             $ROBOT_TESTS_DIR
 
         ROBOT_EXIT_CODE=$?
-
+       
         rebot \
             --outputDir $ROBOT_REPORTS_FINAL_DIR \
-            --merge $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \
-            $ROBOT_REPORTS_FINAL_DIR/output.xml
+            --output $ROBOT_REPORTS_FINAL_DIR/output.xml \
+            --merge $ROBOT_REPORTS_FINAL_DIR/output.xml \
+            ${ROBOT_RERUN_REBOT_OPTIONS} \
+            $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \
+            > /dev/null
+        
+        rm $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml
 
         if [ ${ROBOT_EXIT_CODE} -eq 0 ]
         then

test/rerun.robot

@@ -1,7 +1,25 @@
 *** Test Cases ***
 
-Randomly Fail Test
-    ${rand}=    Evaluate    random.randint(0,3) 
-	IF    ${rand} > 0
+Roll Dice 1
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 2
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 3
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
+        Fail
+	END
+
+Roll Dice 4
+    ${rand}=    Evaluate    random.randint(1,6) 
+	IF    ${rand} > 1
         Fail
 	END