Handle more of the flaky tests (#19193)

handle more of the flaky tests

Author: awaelchli

tests/tests_fabric/utilities/test_distributed.py

@@ -118,6 +118,7 @@ def test_collective_operations(devices, process):
     spawn_launch(process, devices)
 
 
+@pytest.mark.flaky(reruns=3)  # flaky with "process 0 terminated with signal SIGABRT" (GLOO)
 def test_is_shared_filesystem(tmp_path, monkeypatch):
     # In the non-distributed case, every location is interpreted as 'shared'
     assert is_shared_filesystem(SingleDeviceStrategy(torch.device("cpu")))

tests/tests_fabric/utilities/test_spike.py

@@ -28,6 +28,7 @@ def spike_detection_test(fabric, global_rank_spike, spike_value, should_raise):
             )
 
 
+@pytest.mark.flaky(max_runs=3)
 @pytest.mark.parametrize(
     ("global_rank_spike", "num_devices", "spike_value", "finite_only"),
     [

tests/tests_pytorch/callbacks/test_spike.py

@@ -46,6 +46,7 @@ def on_train_batch_end(self, trainer, pl_module, outputs, batch, batch_idx):
             super().on_train_batch_end(trainer, pl_module, outputs, batch, batch_idx)
 
 
+@pytest.mark.flaky(max_runs=3)
 @pytest.mark.parametrize(
     ("global_rank_spike", "num_devices", "spike_value", "finite_only"),
     [

tests/tests_pytorch/models/test_hparams.py

@@ -667,6 +667,8 @@ def test_init_arg_with_runtime_change(tmpdir, cls):
         limit_test_batches=2,
         max_epochs=1,
         logger=TensorBoardLogger(tmpdir),
+        enable_progress_bar=False,
+        enable_checkpointing=False,
     )
     trainer.fit(model)