Use lock to avoid kernel address conflicts in nb-tester (#3143)

This PR uses a lock to try and avoid this error when running in a GitHub
action:

```
zmq.error.ZMQError: Address already in use (addr='tcp://127.0.0.1:45769')
```

This is a bit of a shot in the dark because I can't reproduce it
locally.

This is related to #2387, but is not necessarily a full fix as I believe
there is more than one thing going on there.

***

Inspired by jupyter/nbclient#327

Author: frankharkins

scripts/nb-tester/qiskit_docs_notebook_tester/__init__.py

@@ -32,7 +32,14 @@ async def _main() -> None:
 
     start_time = datetime.now()
     print("Executing notebooks:")
-    results = await asyncio.gather(*(execute_notebook(job) for job in jobs))
+
+    # New kernels choose a port on creation. They usually detect if the port is
+    # in use and will choose another if so, but there can be race conditions
+    # when creating many kernels at once. We use a lock to avoid this.
+    kernel_setup_lock = asyncio.Lock()
+    results = await asyncio.gather(
+        *(execute_notebook(job, kernel_setup_lock) for job in jobs)
+    )
 
     if not args.ignore_trailing_jobs:
         print("Checking for trailing jobs...")

scripts/nb-tester/qiskit_docs_notebook_tester/execute.py

@@ -13,6 +13,7 @@
 from __future__ import annotations
 
 
+import asyncio
 import tempfile
 import textwrap
 from dataclasses import dataclass
@@ -65,7 +66,7 @@ def extract_warnings(notebook: nbformat.NotebookNode) -> list[NotebookWarning]:
     return notebook_warnings
 
 
-async def execute_notebook(job: NotebookJob) -> Result:
+async def execute_notebook(job: NotebookJob, kernel_setup_lock: asyncio.Lock) -> Result:
     """
     Wrapper function for `_execute_notebook` to print status and write result
     """
@@ -80,7 +81,7 @@ async def execute_notebook(job: NotebookJob) -> Result:
         nbclient.exceptions.CellTimeoutError,
     )
     try:
-        nb = await _execute_notebook(job, working_directory.name)
+        nb = await _execute_notebook(job, working_directory.name, kernel_setup_lock)
     except execution_exceptions as err:
         print(f"❌ Problem in {job.path}:\n{err}")
         return Result(False, reason="Exception in notebook")
@@ -118,7 +119,7 @@ async def _execute_in_kernel(kernel: AsyncKernelClient, code: str) -> None:
 
 
 async def _execute_notebook(
-    job: NotebookJob, working_directory: str
+    job: NotebookJob, working_directory: str, kernel_setup_lock: asyncio.Lock
 ) -> nbformat.NotebookNode:
     """
     Use nbclient to execute notebook. The steps are:
@@ -130,11 +131,16 @@ async def _execute_notebook(
     """
     nb = nbformat.read(job.path, as_version=4)
 
-    kernel_manager, kernel = await start_new_async_kernel(
-        kernel_name="python3",
-        extra_arguments=["--InlineBackend.figure_format='svg'"],
-        cwd=working_directory,
-    )
+    async with kernel_setup_lock:
+        # New kernels choose a port on creation. They usually detect if the
+        # port is in use and will choose another if so, but there can be race
+        # conditions when creating many kernels at once.The lock avoids this.
+        # This might be fixed by https://github.com/jupyter/nbclient/pull/327
+        kernel_manager, kernel = await start_new_async_kernel(
+            kernel_name="python3",
+            extra_arguments=["--InlineBackend.figure_format='svg'"],
+            cwd=working_directory,
+        )
 
     await _execute_in_kernel(kernel, job.pre_execute_code)
     if job.backend_patch: