WIP test windows weirdness

Signed-off-by: Jussi Kukkonen <jkukkonen@google.com>

Author: jku

tests/test_updater_ng.py

@@ -359,10 +359,13 @@ def test_user_agent(self) -> None:
 
 class TestParallelUpdater(TestUpdater):
     def test_parallel_updaters(self) -> None:
-        # Refresh two updaters in parallel many times, using the same local metadata cache.
+        # Refresh many updaters in parallel many times, using the same local metadata cache.
         # This should reveal race conditions.
 
-        iterations = 1000
+        # windows on GitHub actions *will* fail with "[Errno 36] Resource deadlock avoided"
+        # if iterations is in the hundreds
+        iterations = 50
+        process_count = 10
 
         # The project root is the parent of the tests directory
         project_root = os.path.dirname(utils.TESTS_DIR)
@@ -376,29 +379,27 @@ def test_parallel_updaters(self) -> None:
             self.metadata_url,
         ]
 
-        p1 = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            cwd=project_root,
-        )
-        p2 = subprocess.Popen(
-            command,
-            stdout=subprocess.PIPE,
-            stderr=subprocess.PIPE,
-            cwd=project_root,
-        )
+        procs = [
+            subprocess.Popen(
+                command,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                cwd=project_root,
+            )
+            for _ in range(process_count)
+        ]
 
-        stdout1, stderr1 = p1.communicate()
-        stdout2, stderr2 = p2.communicate()
+        errout = ""
+        for proc in procs:
+            stdout, stderr = proc.communicate()
+            if proc.returncode != 0:
+                errout += "Parallel Refresh script failed:"
+                errout += f"\nprocess stdout: \n{stdout.decode()}"
+                errout += f"\nprocess stderr: \n{stderr.decode()}"
 
-        if p1.returncode != 0 or p2.returncode != 0:
+        if errout:
             self.fail(
-                "Parallel refresh failed"
-                f"\nprocess 1 stdout: \n{stdout1.decode()}"
-                f"\nprocess 1 stderr: \n{stderr1.decode()}"
-                f"\nprocess 2 stdout: \n{stdout2.decode()}"
-                f"\nprocess 2 stderr: \n{stderr2.decode()}"
+                f"One or more scripts failed parallel refresh test:\n{errout}"
             )
 
 

tuf/ngclient/updater.py

@@ -58,6 +58,7 @@
 import os
 import shutil
 import tempfile
+import time
 from pathlib import Path
 from typing import IO, TYPE_CHECKING, cast
 from urllib import parse
@@ -172,11 +173,33 @@ def _lock_metadata(self) -> Iterator[None]:
         rootdir = Path(self._dir, "root_history")
         rootdir.mkdir(exist_ok=True, parents=True)
 
-        with open(os.path.join(self._dir, ".lock"), "wb") as f:
-            logger.debug("Getting metadata lock...")
-            _lock_file(f)
+        lock_path = os.path.join(self._dir, ".lock")
+        lock_file = None
+        try:
+            logger.warning("Getting metadata lock...")
+            for _ in range(100):  # retry 100 times (10 seconds)
+                try:
+                    # Use 'x+b' for exclusive creation, read/write binary.
+                    # This is atomic: it fails if the file exists.
+                    lock_file = open(lock_path, "x+b")  # noqa: SIM115
+                    break
+                except (FileExistsError, PermissionError):
+                    time.sleep(0.1)
+            else:
+                raise exceptions.RepositoryError(
+                    f"Failed to acquire lock {lock_path} after 10 seconds"
+                )
+
+            _lock_file(lock_file)
             yield
-            logger.debug("Releasing metadata lock")
+
+        finally:
+            if lock_file is not None:
+                lock_file.close()
+            # The lock is released by deleting the file.
+            with contextlib.suppress(FileNotFoundError):
+                os.remove(lock_path)
+            logger.warning("Released metadata lock")
 
     def refresh(self) -> None:
         """Refresh top-level metadata.