Fix CI timeouts (#4458)

* Do not use 'mpiexec -n 1' for serial tests

This was causing a hang during MPI_Finalize.

* modify timeouts

Author: connorjward

.github/workflows/core.yml

@@ -233,7 +233,7 @@ jobs:
           : # Use pytest-xdist here so we can have a single collated output (not possible
           : # for parallel tests)
           firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/tsfc
-        timeout-minutes: 60
+        timeout-minutes: 10
 
       - name: Run PyOP2 tests
         if: inputs.run_tests && (success() || steps.install.conclusion == 'success')
@@ -245,7 +245,7 @@ jobs:
           firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2
           firedrake-run-split-tests 3 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2
           firedrake-run-split-tests 4 2 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/pyop2
-        timeout-minutes: 15
+        timeout-minutes: 10
 
 
       - name: Run Firedrake tests (nprocs = 1)
@@ -255,14 +255,14 @@ jobs:
           : # Use pytest-xdist here so we can have a single collated output (not possible
           : # for parallel tests)
           firedrake-run-split-tests 1 1 -n 8 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
-        timeout-minutes: 60
+        timeout-minutes: 90
 
       - name: Run tests (nprocs = 2)
         if: inputs.run_tests && (success() || steps.install.conclusion == 'success')
         run: |
           . venv/bin/activate
           firedrake-run-split-tests 2 4 "$EXTRA_PYTEST_ARGS" firedrake-repo/tests/firedrake
-        timeout-minutes: 30
+        timeout-minutes: 60
 
       - name: Run tests (nprocs = 3)
         if: inputs.run_tests && (success() || steps.install.conclusion == 'success')

scripts/firedrake-run-split-tests

@@ -35,9 +35,14 @@ extra_args=${@:3}
 cache_cmd="PYOP2_CACHE_DIR=\$VIRTUAL_ENV/.cache/pyop2/job{#} \
            FIREDRAKE_TSFC_KERNEL_CACHE_DIR=\$VIRTUAL_ENV/.cache/tsfc/job{#}"
 
-pytest_exec="mpiexec -n ${num_procs} python -m pytest"
+if [ $num_procs = 1 ]; then
+    # Cannot use mpiexec -n 1 because this can sometimes hang with
+    # OpenMPI at MPI_Finalize
+    pytest_exec="python3 -m pytest"
+else
+    pytest_exec="mpiexec -n ${num_procs} python3 -m pytest"
+fi
 marker_spec="parallel[${num_procs}]"
-
 pytest_cmd="${pytest_exec} -v \
             --splits ${num_jobs} --group {#} \
             -m ${marker_spec} ${extra_args}"