debug prints

Author: apbose

.github/workflows/build-test-linux-x86_64.yml

@@ -519,7 +519,7 @@ jobs:
 
   L2-dynamo-distributed-tests:
     name: L2 dynamo distributed tests
-    needs: [filter-matrix, build, L1-dynamo-core-tests, L1-dynamo-compile-tests, L1-torch-compile-tests, L1-torchscript-tests]
+    needs: [filter-matrix, build]
     strategy:
       fail-fast: false
       matrix:
@@ -568,17 +568,36 @@ jobs:
 
         # Install MPI (required for TensorRT-LLM plugins)
         echo "Installing MPI..."
-        dnf install -y mpich mpich-devel openmpi openmpi-devel
+        dnf install -y openmpi openmpi-devel
+        
+        # Add OpenMPI to PATH (RHEL/AlmaLinux specific location)
+        export PATH="/usr/lib64/openmpi/bin:$PATH"
+        export LD_LIBRARY_PATH="/usr/lib64/openmpi/lib:$LD_LIBRARY_PATH"
+        
+        # Verify mpirun is accessible
+        which mpirun
+        mpirun --version
 
         # Run distributed tests
         pushd .
         cd tests/py/dynamo
 
         echo "Running distributed tests with mpirun..."
+        echo "[CONFIG] Number of GPUs to use: ${{ matrix.num_gpus }}"
+        echo "[AVAILABLE] GPUs detected by nvidia-smi: $(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)"
+        
+        # Use a wrapper script to ensure only rank 0 writes the JUnit XML
+        # Each rank runs pytest, but only rank 0 saves results to avoid file conflicts
+        RANK_0_XML="${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_distributed_test_results.xml"
         mpirun --allow-run-as-root -n ${{ matrix.num_gpus }} \
-          python -m pytest -ra \
-          --junitxml=${RUNNER_TEST_RESULTS_DIR}/l2_dynamo_distributed_test_results.xml \
-          distributed/test_nccl_ops.py
+          bash -c '
+            echo "[MPI DEBUG] Rank: ${OMPI_COMM_WORLD_RANK:-0}, World Size: ${OMPI_COMM_WORLD_SIZE:-1}"
+            if [ "${OMPI_COMM_WORLD_RANK:-0}" -eq 0 ]; then
+              python -m pytest -ra --junitxml='"${RANK_0_XML}"' distributed/test_nccl_ops.py
+            else
+              python -m pytest -ra distributed/test_nccl_ops.py
+            fi
+          '
 
         popd