fix: Remove fail-fast (-x) and guard distributed teardown against deadlock (#4139)

Signed-off-by: oliver könig <okoenig@nvidia.com>
Co-authored-by: Claude Sonnet 4.6 <noreply@anthropic.com>

Author: ko3n1g

pyproject.toml

@@ -234,7 +234,7 @@ exclude = '''
 '''
 
 [tool.pytest.ini_options]
-addopts = "--durations=15 -s -rA -x"
+addopts = "--durations=15 -s -rA"
 testpaths = ["tests"]
 python_files = "test_*.py"
 markers = [

tests/unit_tests/conftest.py

@@ -1,6 +1,7 @@
 # Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 import os
+from datetime import timedelta
 from pathlib import Path
 
 import pytest
@@ -42,7 +43,10 @@ def pytest_sessionfinish(session, exitstatus):
 def cleanup():
     yield
     if torch.distributed.is_initialized():
-        torch.distributed.barrier()
+        try:
+            torch.distributed.barrier(timeout=timedelta(seconds=30))
+        except Exception:
+            return
         torch.distributed.destroy_process_group()
 
 

tests/unit_tests/run_ci_test.sh

@@ -148,14 +148,14 @@ for i in $(seq $UNIT_TEST_REPEAT); do
         --data-file=.coverage.unit_tests \
         --source=megatron/core \
         -m pytest \
-        -xvs \
+        -vs \
         ${IGNORE_ARGS[@]} \
         -m "'not experimental and ${MARKER_ARG}'" $(echo "$BUCKET" | sed 's|/\*\*/\*\.py$||'))
     eval "$CMD"
 
     if [[ "$TAG" == "latest" ]]; then
         CMD=$(echo uv run --no-sync python -m torch.distributed.run ${DISTRIBUTED_ARGS[@]} -m pytest \
-            -xvs \
+            -vs \
             --experimental \
              ${IGNORE_ARGS[@]} \
             -m "'experimental and ${MARKER_ARG}'" $(echo "$BUCKET" | sed 's|/\*\*/\*\.py$||'))

tests/unit_tests/test_utilities.py

@@ -91,7 +91,11 @@ def destroy_model_parallel():
         os.environ.pop('NVTE_UNFUSED_ATTN', None)
         if not Utils.inited:
             return
-        torch.distributed.barrier()
+        try:
+            torch.distributed.barrier(timeout=timedelta(seconds=30))
+        except Exception:
+            Utils.inited = False
+            return
         ps.destroy_model_parallel()
         Utils.inited = False