Add support for retrying when GitHub rate limit the import

Author: PeterJCLaw

scripts/import_backends.py

@@ -6,6 +6,7 @@
 from typing import TYPE_CHECKING, Dict, List, Sequence, cast
 
 import github
+import retrying
 from termcolor import cprint
 from ticket_type import Ticket
 
@@ -224,12 +225,41 @@ def submit(self, ticket: Ticket) -> int:
         # larger gap for better reliability.
         time.sleep(5)
 
-        issue = self.repo.create_issue(
-            ticket.summary,
-            self.description_text(ticket),
-            milestone=self.get_or_create_milestone(ticket.milestone),
-            labels=self.labels(ticket),
+        # We also want to wait and retry when GitHub then additionally rate
+        # limit us anyway...
+
+        def retry_on_exception(exception: Exception) -> bool:
+            return isinstance(exception, github.GithubException)
+
+        @retrying.retry(
+            retry_on_exception=retry_on_exception,
+            wait_fixed=20_000,
         )
+        def create_issue() -> github.Issue.Issue:
+            RATE_LIMIT_MESSAGE = "exceeded a secondary rate limit and have been temporarily blocked"  # noqa:E501
+
+            try:
+                return self.repo.create_issue(
+                    ticket.summary,
+                    self.description_text(ticket),
+                    milestone=self.get_or_create_milestone(ticket.milestone),
+                    labels=self.labels(ticket),
+                )
+            except github.GithubException as e:
+                message = e.data.get('message')
+                if (
+                    isinstance(message, str)
+                    and RATE_LIMIT_MESSAGE in message
+                ):
+                    print("... GitHub Rate Limited ...")
+                    raise github.RateLimitExceededException(
+                        status=e.status,
+                        data=e.data,
+                        headers=e.headers,
+                    ) from e
+                raise
+
+        issue = create_issue()
 
         ticket_number: int = issue.number
         self._known_titles[ticket_number] = ticket.summary

scripts/requirements-dev.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile ./scripts/requirements-dev.in
 #
@@ -87,8 +87,13 @@ requests==2.27.1
     # via
     #   -r ./scripts/requirements.txt
     #   pygithub
+retrying==1.3.3
+    # via -r ./scripts/requirements.txt
 six==1.16.0
-    # via flake8-tuple
+    # via
+    #   -r ./scripts/requirements.txt
+    #   flake8-tuple
+    #   retrying
 termcolor==1.1.0
     # via -r ./scripts/requirements.txt
 testfixtures==6.18.5

scripts/requirements.in

@@ -1,3 +1,4 @@
 PyGithub
 PyYAML
+retrying
 termcolor

scripts/requirements.txt

@@ -1,6 +1,6 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
-# To update, run:
+# This file is autogenerated by pip-compile with Python 3.8
+# by the following command:
 #
 #    pip-compile ./scripts/requirements.in
 #
@@ -26,6 +26,10 @@ pyyaml==6.0
     # via -r ./scripts/requirements.in
 requests==2.27.1
     # via pygithub
+retrying==1.3.3
+    # via -r ./scripts/requirements.in
+six==1.16.0
+    # via retrying
 termcolor==1.1.0
     # via -r ./scripts/requirements.in
 urllib3==1.26.9