Add a retry for running tests in nightly action

Author: antonwolfy

.github/workflows/conda-package.yml

@@ -188,8 +188,10 @@ jobs:
           retry_on: any
           command: |
             . $CONDA/etc/profile.d/conda.sh
-            conda activate ${{ env.TEST_ENV_NAME }}
-            pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.TEST_ENV_NAME }}
+
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
 
   test_windows:
     name: Test ['windows-2019', python='${{ matrix.python }}']
@@ -323,7 +325,7 @@ jobs:
           retry_on: any
           command: >-
             mamba activate ${{ env.TEST_ENV_NAME }}
-            & pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+            & python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
 
   upload:
     name: Upload ['${{ matrix.os }}', python='${{ matrix.python }}']

.github/workflows/cron-run-tests.yaml

@@ -15,6 +15,8 @@ env:
   PACKAGE_NAME: dpnp
   CHANNELS: '-c dppy/label/dev -c https://software.repos.intel.com/python/conda/ -c conda-forge --override-channels'
   TEST_ENV_NAME: test
+  RERUN_TESTS_ON_FAILURE: 'true'
+  RUN_TESTS_MAX_ATTEMPTS: 2
 
 jobs:
   test:
@@ -82,7 +84,40 @@ jobs:
           python -c "import dpnp; print(dpnp.__version__)"
 
       - name: Run tests
+        if: env.RERUN_TESTS_ON_FAILURE != 'true'
         run: |
           python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
         env:
           SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tests on Linux
+        if: env.RERUN_TESTS_ON_FAILURE == 'true' && matrix.runner != 'windows-2019'
+        id: run_tests_linux
+        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+        with:
+          timeout_minutes: 10
+          max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
+          retry_on: any
+          command: |
+            . $CONDA/etc/profile.d/conda.sh
+            . $CONDA/etc/profile.d/mamba.sh
+            mamba activate ${{ env.TEST_ENV_NAME }}
+
+            echo "SYCL_CACHE_PERSISTENT=$SYCL_CACHE_PERSISTENT"
+
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+        env:
+          SYCL_CACHE_PERSISTENT: 1
+
+      - name: ReRun tests on Windows
+        if: env.RERUN_TESTS_ON_FAILURE == 'true' && matrix.runner == 'windows-2019'
+        id: run_tests_win
+        uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e # v3.0.0
+        with:
+          timeout_minutes: 15
+          max_attempts: ${{ env.RUN_TESTS_MAX_ATTEMPTS }}
+          retry_on: any
+          command: |
+            python -m pytest -ra --pyargs ${{ env.PACKAGE_NAME }}.tests
+        env:
+          SYCL_CACHE_PERSISTENT: 1