test: addressing flaky spawn "process 0 terminated with signal SIGABRT" (#20933)

Author: Borda

.github/workflows/ci-tests-fabric.yml

@@ -167,7 +167,7 @@ jobs:
         run: |
           echo $GITHUB_RUN_ID
           python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \
-            -m pytest -v --timeout=30 --durations=50 --random-order-seed=$GITHUB_RUN_ID \
+            -m pytest -v --timeout=60 --durations=50 --random-order-seed=$GITHUB_RUN_ID \
             --junitxml=junit.xml -o junit_family=legacy  # NOTE: for Codecov's test results
 
       - name: Statistics

.github/workflows/ci-tests-pytorch.yml

@@ -196,7 +196,7 @@ jobs:
         run: |
           echo $GITHUB_RUN_ID
           python -m coverage run --source ${{ env.COVERAGE_SCOPE }} \
-            -m pytest . -v --timeout=60 --durations=50 --random-order-seed=$GITHUB_RUN_ID \
+            -m pytest . -v --timeout=90 --durations=50 --random-order-seed=$GITHUB_RUN_ID \
             --junitxml=junit.xml -o junit_family=legacy  # NOTE: for Codecov's test results
 
       - name: Statistics

tests/tests_pytorch/callbacks/test_stochastic_weight_avg.py

@@ -354,6 +354,8 @@ def test_swa_resume_training_from_checkpoint_custom_scheduler(tmp_path, crash_on
 
 
 @RunIf(skip_windows=True)
+# flaky with "process 0 terminated with signal SIGABRT" (GLOO)
+@pytest.mark.flaky(reruns=3, only_rerun="torch.multiprocessing.spawn.ProcessExitedException")
 def test_swa_resume_training_from_checkpoint_ddp(tmp_path):
     model = SwaTestModel(crash_on_epoch=3)
     resume_model = SwaTestModel()