diff --git a/.github/workflows/docker-build.yml b/.github/workflows/docker-build.yml index 1160b84..ab7ba30 100644 --- a/.github/workflows/docker-build.yml +++ b/.github/workflows/docker-build.yml @@ -154,3 +154,9 @@ jobs: -e BROWSER=$BROWSER \ -e TZ="America/New_York" \ $IMAGE_NAME + - name: Rerun Test + run: | + podman run --user=2000:2000 --shm-size=1g \ + -v `pwd`/test/rerun.robot:/opt/robotframework/tests/rerun.robot:Z \ + -e ROBOT_RERUN_FAILED=20 \ + $IMAGE_NAME diff --git a/Dockerfile b/Dockerfile index 8776d4c..8d9ef85 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,6 +15,9 @@ ENV ROBOT_TESTS_DIR /opt/robotframework/tests # Set the working directory environment variable ENV ROBOT_WORK_DIR /opt/robotframework/temp +# Set the maximum number of rounds to rerun failed tests +ENV ROBOT_RERUN_FAILED 0 + # Setup X Window Virtual Framebuffer ENV SCREEN_COLOUR_DEPTH 24 ENV SCREEN_HEIGHT 1080 @@ -55,11 +58,6 @@ ENV XVFB_VERSION 21.1.18 # By default, no reports are uploaded to AWS S3 ENV AWS_UPLOAD_TO_S3 false -# Prepare binaries to be executed -COPY bin/chromedriver.sh /opt/robotframework/drivers/chromedriver -COPY bin/chrome.sh /opt/robotframework/bin/chrome -COPY bin/run-tests-in-virtual-screen.sh /opt/robotframework/bin/ - # Install system dependencies RUN dnf upgrade -y --refresh \ && dnf install -y \ @@ -131,6 +129,11 @@ ENV PATH=/opt/microsoft/msedge:$PATH # Additionally, it cannot run fully on any OS due to https://github.com/microsoft/playwright/issues/29559 RUN rfbrowser init chromium firefox +# Prepare binaries to be executed +COPY bin/chromedriver.sh /opt/robotframework/drivers/chromedriver +COPY bin/chrome.sh /opt/robotframework/bin/chrome +COPY bin/run-tests-in-virtual-screen.sh /opt/robotframework/bin/ + # Create the default report and work folders with the default user to avoid runtime issues # These folders are writeable by anyone, to ensure the user can be changed on the command line. RUN mkdir -p ${ROBOT_REPORTS_DIR} \ diff --git a/README.md b/README.md index e7e2f28..b4c5f29 100644 --- a/README.md +++ b/README.md @@ -199,6 +199,20 @@ rpa==1.50.0 **For large dependencies, it is still recommended to extend the project's image and to add them there, to avoid delaying the CI/CD pipelines with repeated dependency installations.** +### Rerunning tests + +Failing tests can be rerun by setting the environment variable `ROBOT_RERUN_FAILED` to a value above 0. All reruns of failed tests are executed without parallelization. +The number in environment variable `ROBOT_RERUN_FAILED` dictates how many rerun-rounds are made at maximum. All rerun-rounds will only test what failed in the previous round. +The report files combine the results of all rounds, the last round providing the final result. + +The default value for `ROBOT_RERUN_FAILED` is 0, meaning that tests will not be executed again if they fail. + +```sh +docker run \ + -e ROBOT_RERUN_FAILED=1 \ + ppodgorsek/robot-framework:latest +``` + ## Security consideration diff --git a/bin/run-tests-in-virtual-screen.sh b/bin/run-tests-in-virtual-screen.sh index 299b654..78daa11 100755 --- a/bin/run-tests-in-virtual-screen.sh +++ b/bin/run-tests-in-virtual-screen.sh @@ -53,6 +53,34 @@ fi ROBOT_EXIT_CODE=$? +if [[ ${ROBOT_EXIT_CODE} -gt 0 ]] +then + for ((i = 0 ; i < ${ROBOT_RERUN_FAILED} ; i++ )) + do + echo "Rerunning failed tests, round ${i}..." + xvfb-run \ + --server-args="-screen 0 ${SCREEN_WIDTH}x${SCREEN_HEIGHT}x${SCREEN_COLOUR_DEPTH} -ac" \ + robot \ + --rerunfailed $ROBOT_REPORTS_FINAL_DIR/output.xml \ + --output $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \ + --outputDir $ROBOT_REPORTS_FINAL_DIR \ + ${ROBOT_OPTIONS} \ + $ROBOT_TESTS_DIR + + ROBOT_EXIT_CODE=$? + + rebot \ + --outputDir $ROBOT_REPORTS_FINAL_DIR \ + --merge $ROBOT_REPORTS_FINAL_DIR/output_rerun.xml \ + $ROBOT_REPORTS_FINAL_DIR/output.xml + + if [ ${ROBOT_EXIT_CODE} -eq 0 ] + then + break + fi + done +fi + if [ ${AWS_UPLOAD_TO_S3} = true ] then echo "Uploading report to AWS S3..." diff --git a/test/rerun.robot b/test/rerun.robot new file mode 100644 index 0000000..33a95d2 --- /dev/null +++ b/test/rerun.robot @@ -0,0 +1,7 @@ +*** Test Cases *** + +Randomly Fail Test + ${rand}= Evaluate random.randint(0,3) + IF ${rand} > 0 + Fail + END \ No newline at end of file