repo_synchroniser: retry all remote operations (bug 1982268) (#73)

* retry: return callback's return
* repo_synchroniser: retry all remote operations (bug 1982268)

Otherwise we may needlessly fail the message on transient errors.

Author: shtrom

git_hg_sync/repo_synchronizer.py

@@ -56,10 +56,13 @@ def fetch_all_from_remote(
         self, repo: Repo, remote: str, verbose: bool = False
     ) -> None:
         try:
-            self._log_git_execute(
-                repo,
-                ["git", "-c", "cinnabar.graft=true", "fetch", "--tags", remote],
-                verbose,
+            retry(
+                f"fetching changes and tags from {remote}",
+                lambda: self._log_git_execute(
+                    repo,
+                    ["git", "-c", "cinnabar.graft=true", "fetch", "--tags", remote],
+                    verbose,
+                ),
             )
 
         except GitCommandError as exc:
@@ -68,7 +71,12 @@ def fetch_all_from_remote(
                 raise exc
 
         if remote.startswith("hg::"):
-            self._log_git_execute(repo, ["git", "cinnabar", "fetch", "--tags"], verbose)
+            retry(
+                f"fetching Hg tags with cinnabar from {remote}",
+                lambda: self._log_git_execute(
+                    repo, ["git", "cinnabar", "fetch", "--tags"], verbose
+                ),
+            )
 
     @staticmethod
     def _log_git_execute(repo: Repo, command: list[str], verbose: bool = False) -> None:
@@ -173,15 +181,20 @@ def sync(
             # If the destination branch is not present locally, but exists remotely, we
             # explicitly fetch it.
             local_branch_exists = repo.git.branch("-l", tag_branch)
-            remote_branch_exists = repo.git.execute(
-                [
-                    "git",
-                    "ls-remote",
-                    destination_remote,
-                    self._cinnabar_branch(tag_branch),
-                ],
-                stdout_as_string=True,
+            remote_branch_exists = retry(
+                "checking if tag branch already exists remotely",
+                partial(
+                    repo.git.execute,
+                    [
+                        "git",
+                        "ls-remote",
+                        destination_remote,
+                        self._cinnabar_branch(tag_branch),
+                    ],
+                    stdout_as_string=True,
+                ),
             )
+
             if not local_branch_exists and remote_branch_exists:
                 retry(
                     f"fetching existing tag branch from {destination_remote}",
@@ -247,9 +260,13 @@ def sync(
             push_args = [destination_remote, ref]
             # Force-push the branch if it doesn't exist on the remote yet.
             # This is necessary to create new branches, more specifically for tags.
-            if not repo.git.execute(
-                ["git", "ls-remote", destination_remote, ref.split(":")[1]],
-                stdout_as_string=True,
+            if not retry(
+                "checking if destination branch needs to be created",
+                partial(
+                    repo.git.execute,
+                    ["git", "ls-remote", destination_remote, ref.split(":")[1]],
+                    stdout_as_string=True,
+                ),
             ):
                 push_args = ["-f"] + push_args
             logger.debug(f"Push arguments: {push_args}")

git_hg_sync/retry.py

@@ -1,5 +1,6 @@
 import time
 from collections.abc import Callable
+from typing import Any
 
 import sentry_sdk
 from mozlog import get_proxy_logger
@@ -13,7 +14,7 @@ def retry(
     *,
     tries: int = 2,
     delay: float = 0.25,
-) -> None:
+) -> Any:
     """Run a `callback` up to `tries` times with a `delay` between Exceptions.
 
     The `callback` can be a closure built from a lambda function if needed.
@@ -42,8 +43,7 @@ def retry(
     logger.debug(action)
     for attempt in range(1, tries + 1):
         try:
-            callback()
-            break
+            return callback()
         except Exception as exc:
             sentry_sdk.capture_exception(exc)
             action_text = f" {action}" if action else ""
@@ -59,3 +59,7 @@ def retry(
                     exc_info=True,
                 )
                 raise
+
+    raise Exception(
+        "The retryable callback should have either succeeded or raised, so we should not be here."
+    )