From 1406aece3e7d1cd7f0c83d1d0544753563fd43f1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Salom=C3=A4ki?= Date: Tue, 9 Sep 2025 21:28:00 +0300 Subject: [PATCH 1/3] Update Firefox to version 142.0 --- Dockerfile | 2 +- README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 8d9ef85..c71c0e7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -44,7 +44,7 @@ ENV DATADRIVER_VERSION 1.11.2 ENV DATETIMETZ_VERSION 1.0.6 ENV MICROSOFT_EDGE_VERSION 139.0.3405.86 ENV FAKER_VERSION 6.0.0 -ENV FIREFOX_VERSION 141.0 +ENV FIREFOX_VERSION 142.0 ENV FTP_LIBRARY_VERSION 1.9 ENV GECKO_DRIVER_VERSION v0.36.0 ENV IMAP_LIBRARY_VERSION 0.4.11 diff --git a/README.md b/README.md index b4c5f29..6025db9 100644 --- a/README.md +++ b/README.md @@ -63,7 +63,7 @@ The versions used are: * [Robot Framework SeleniumLibrary](https://github.com/robotframework/SeleniumLibrary) 6.7.1 * [Robot Framework SSHLibrary](https://github.com/robotframework/SSHLibrary) 3.8.0 * [Axe Selenium Library](https://github.com/mozilla-services/axe-selenium-python) 2.1.6 -* Firefox 141.0 +* Firefox 142.0 * [Chrome for Testing](https://googlechromelabs.github.io/chrome-for-testing/) 139.0 * Microsoft Edge 139.0 * [Amazon AWS CLI](https://pypi.org/project/awscli/) 1.42.11 From 1d36b74e95a5ec0e8d809a9ce5bcdf71cfaeaf2c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Salom=C3=A4ki?= Date: Wed, 10 Sep 2025 00:20:35 +0300 Subject: [PATCH 2/3] Fix reporting on rerun-tests * Echo rerun-rounds starting from 1, hide rebot stdout * Zip log-files from previous test-rounds --- .github/workflows/docker-build.yml | 2 +- Dockerfile | 3 +++ README.md | 1 + bin/run-tests-in-virtual-screen.sh | 19 ++++++++++++++----- test/rerun.robot | 24 +++++++++++++++++++++--- 5 files changed, 40 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index ab7ba30..456f2ca 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -158,5 +158,5 @@ jobs: run: | podman run --user=2000:2000 --shm-size=1g \ -v `pwd`/test/rerun.robot:/opt/robotframework/tests/rerun.robot:Z \ - -e ROBOT_RERUN_FAILED=20 \ + -e ROBOT_RERUN_FAILED=50 \ $IMAGE_NAME diff --git a/Dockerfile b/Dockerfile index c71c0e7..8c3ce8e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,6 +18,9 @@ ENV ROBOT_WORK_DIR /opt/robotframework/temp # Set the maximum number of rounds to rerun failed tests ENV ROBOT_RERUN_FAILED 0 +# Options to the rebot command when rerunning failed tests +ENV ROBOT_RERUN_REBOT_OPTIONS "" + # Setup X Window Virtual Framebuffer ENV SCREEN_COLOUR_DEPTH 24 ENV SCREEN_HEIGHT 1080 diff --git a/README.md b/README.md index 6025db9..4ac89cf 100644 --- a/README.md +++ b/README.md @@ -14,6 +14,7 @@ * [Testing emails](#testing-emails) * [Dealing with Datetimes and Timezones](#dealing-with-datetimes-and-timezones) * [Installing additional dependencies](#installing-additional-dependencies) + * [Rerunning tests](#rerunning-tests) * [Security consideration](#security-consideration) * [Continuous integration](#continuous-integration) * [Azure DevOps pipeline](#ci-azure-devops) diff --git a/bin/run-tests-in-virtual-screen.sh b/bin/run-tests-in-virtual-screen.sh index 78daa11..a303146 100755 --- a/bin/run-tests-in-virtual-screen.sh +++ b/bin/run-tests-in-virtual-screen.sh @@ -53,11 +53,15 @@ fi ROBOT_EXIT_CODE=$? -if [[ ${ROBOT_EXIT_CODE} -gt 0 ]] +if [ ${ROBOT_EXIT_CODE} -gt 0 ] then for ((i = 0 ; i < ${ROBOT_RERUN_FAILED} ; i++ )) do - echo "Rerunning failed tests, round ${i}..." + echo "Rerunning failed tests, round $((i+1))..." + + # Store previous run into a tarball as screenshots etc from the previous run are removed by rebot + (cd $ROBOT_REPORTS_FINAL_DIR && tar czf report_run_${i}.tar.gz browser *.html *.xml *.log *.png) + xvfb-run \ --server-args="-screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_COLOUR_DEPTH} -ac" \ robot \ @@ -68,11 +72,16 @@ then $ROBOT_TESTS_DIR ROBOT_EXIT_CODE=$? - + rebot \ --outputDir $ROBOT_REPORTS_FINAL_DIR \ - --merge $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \ - $ROBOT_REPORTS_FINAL_DIR/output.xml + --output $ROBOT_REPORTS_FINAL_DIR/output.xml \ + --merge $ROBOT_REPORTS_FINAL_DIR/output.xml \ + ${ROBOT_RERUN_REBOT_OPTIONS} \ + $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \ + > /dev/null + + rm $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml if [ ${ROBOT_EXIT_CODE} -eq 0 ] then diff --git a/test/rerun.robot b/test/rerun.robot index 33a95d2..c1b66cd 100644 --- a/test/rerun.robot +++ b/test/rerun.robot @@ -1,7 +1,25 @@ *** Test Cases *** -Randomly Fail Test - ${rand}= Evaluate random.randint(0,3) - IF ${rand} > 0 +Roll Dice 1 + ${rand}= Evaluate random.randint(1,6) + IF ${rand} > 1 + Fail + END + +Roll Dice 2 + ${rand}= Evaluate random.randint(1,6) + IF ${rand} > 1 + Fail + END + +Roll Dice 3 + ${rand}= Evaluate random.randint(1,6) + IF ${rand} > 1 + Fail + END + +Roll Dice 4 + ${rand}= Evaluate random.randint(1,6) + IF ${rand} > 1 Fail END \ No newline at end of file From 6e583d63c564cd13edcbf8fa29454f569d6dc8ae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ville=20Salom=C3=A4ki?= Date: Fri, 12 Sep 2025 22:21:30 +0300 Subject: [PATCH 3/3] Rename ROBOT_RERUN_FAILED to ROBOT_RERUN_MAX_ROUNDS --- .github/workflows/docker-build.yml | 5 ++--- Dockerfile | 2 +- README.md | 8 ++++---- bin/run-tests-in-virtual-screen.sh | 2 +- 4 files changed, 8 insertions(+), 9 deletions(-) diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 456f2ca..1762294 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -33,8 +33,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Build image - run: - podman build . --tag $IMAGE_NAME --file Dockerfile + run: podman build . --tag $IMAGE_NAME --file Dockerfile - name: Basic Test run: | podman run --shm-size=1g \ @@ -158,5 +157,5 @@ jobs: run: | podman run --user=2000:2000 --shm-size=1g \ -v `pwd`/test/rerun.robot:/opt/robotframework/tests/rerun.robot:Z \ - -e ROBOT_RERUN_FAILED=50 \ + -e ROBOT_RERUN_MAX_ROUNDS=50 \ $IMAGE_NAME diff --git a/Dockerfile b/Dockerfile index 8c3ce8e..d938306 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ ENV ROBOT_TESTS_DIR /opt/robotframework/tests ENV ROBOT_WORK_DIR /opt/robotframework/temp # Set the maximum number of rounds to rerun failed tests -ENV ROBOT_RERUN_FAILED 0 +ENV ROBOT_RERUN_MAX_ROUNDS 0 # Options to the rebot command when rerunning failed tests ENV ROBOT_RERUN_REBOT_OPTIONS "" diff --git a/README.md b/README.md index 4ac89cf..697236a 100644 --- a/README.md +++ b/README.md @@ -202,15 +202,15 @@ rpa==1.50.0 ### Rerunning tests -Failing tests can be rerun by setting the environment variable `ROBOT_RERUN_FAILED` to a value above 0. All reruns of failed tests are executed without parallelization. -The number in environment variable `ROBOT_RERUN_FAILED` dictates how many rerun-rounds are made at maximum. All rerun-rounds will only test what failed in the previous round. +Failing tests can be rerun by setting the environment variable `ROBOT_RERUN_MAX_ROUNDS` to a value above 0. All reruns of failed tests are executed without parallelization. +The number in environment variable `ROBOT_RERUN_MAX_ROUNDS` dictates how many rerun-rounds are made at maximum. All rerun-rounds will only test what failed in the previous round. The report files combine the results of all rounds, the last round providing the final result. -The default value for `ROBOT_RERUN_FAILED` is 0, meaning that tests will not be executed again if they fail. +The default value for `ROBOT_RERUN_MAX_ROUNDS` is 0, meaning that tests will not be executed again if they fail. ```sh docker run \ - -e ROBOT_RERUN_FAILED=1 \ + -e ROBOT_RERUN_MAX_ROUNDS=1 \ ppodgorsek/robot-framework:latest ``` diff --git a/bin/run-tests-in-virtual-screen.sh b/bin/run-tests-in-virtual-screen.sh index a303146..3cf64b4 100755 --- a/bin/run-tests-in-virtual-screen.sh +++ b/bin/run-tests-in-virtual-screen.sh @@ -55,7 +55,7 @@ ROBOT_EXIT_CODE=$? if [ ${ROBOT_EXIT_CODE} -gt 0 ] then - for ((i = 0 ; i < ${ROBOT_RERUN_FAILED} ; i++ )) + for ((i = 0 ; i < ${ROBOT_RERUN_MAX_ROUNDS} ; i++ )) do echo "Rerunning failed tests, round $((i+1))..."