Use spawn start method for multiprocessing inside the test to avoid deadlock/timeout on Azure (#5213)

* Fixes #5191 
* use a spawn start method inside the test instead of pool to avoid deadlock (especially observed on Azure)
* add duration for azure
* add  explicit comments to code

Author: yuxuanzhuang

azure-pipelines.yml

@@ -131,7 +131,7 @@ jobs:
     displayName: 'Check installed packages'
   - powershell: |
       cd testsuite
-      pytest MDAnalysisTests --disable-pytest-warnings -n logical --timeout=200 -rsx --cov=MDAnalysis
+      pytest MDAnalysisTests --disable-pytest-warnings -n logical --timeout=200 -rsx --cov=MDAnalysis --durations=50
     displayName: 'Run MDAnalysis Test Suite'
   - script: |
       curl -s https://codecov.io/bash | bash

testsuite/MDAnalysisTests/parallelism/test_multiprocessing.py

@@ -129,12 +129,19 @@ def cog(u, ag, frame_id):
 
 def test_multiprocess_COG(u):
     ag = u.atoms[2:5]
-
     ref = np.array([cog(u, ag, i) for i in range(3)])
 
-    p = multiprocessing.Pool(2)
-    res = np.array([p.apply(cog, args=(u, ag, i)) for i in range(3)])
-    p.close()
+    # Explicitly use `spawn` for multiprocessing tests
+    # As `fork` can lead to deadlock in pytest because each xdist worker creates a multiprocessing.Pool.
+    # related issue: #5191; PR #5213
+    ctx = multiprocessing.get_context("spawn")
+    # The commented code can lead to timeout that is not marked as a failure due.
+    # p = multiprocessing.Pool(2)
+    # res = np.array([p.apply(cog, args=(u, ag, i)) for i in range(3)])
+    # p.close()
+    with ctx.Pool(2) as p:
+        res = np.array(p.starmap(cog, [(u, ag, i) for i in range(3)]))
+
     assert_equal(ref, res)
 
 
@@ -147,10 +154,11 @@ def test_universe_unpickle_in_new_process():
     u = mda.Universe(GRO, XTC)
     ref = [getnames(u, i) for i in range(3)]
 
-    p = multiprocessing.Pool(2)
-    res = [p.apply(getnames, args=(u, i)) for i in range(3)]
-    p.close()
-
+    # related issue: #5191; PR #5213
+    ctx = multiprocessing.get_context("spawn")
+    with ctx.Pool(2) as p:
+        res = [p.apply_async(getnames, args=(u, i)) for i in range(3)]
+        res = [r.get() for r in res]
     assert_equal(ref, res)
 
 
@@ -162,7 +170,10 @@ def test_creating_multiple_universe_without_offset(temp_xtc, ncopies=3):
     #  a problem (see PR #3375 and issues #3230, #1988)
 
     args = (GRO, str(temp_xtc))
-    with multiprocessing.Pool(2) as p:
+
+    # related issue: #5191; PR #5213
+    ctx = multiprocessing.get_context("spawn")
+    with ctx.Pool(2) as p:
         universes = [p.apply_async(mda.Universe, args) for i in range(ncopies)]
         universes = [universe.get() for universe in universes]