CLOUDDST-28603: Add retry on ErrataClient._call_et (#666)

This commit adds a retry on `ErrataClientBase._call_et` in order to
re-attempt failed requests with the Errata Tool.

Refers to CLOUDDST-28603

---------

Signed-off-by: Jonathan Gangi <[email protected]>

Author: JAVGan

src/pushsource/_impl/backend/errata_source/errata_client.py

@@ -7,12 +7,14 @@
 import threading
 import xmlrpc.client as xmlrpc_client  # nosec B411
 from urllib.parse import urljoin
+from urllib3.util.retry import Retry
 import warnings
 
 import gssapi
 from more_executors import Executors
 from more_executors.futures import f_zip, f_map
 import requests
+import requests.adapters
 import requests_gssapi
 
 from ...compat_attr import attr
@@ -21,6 +23,14 @@
 
 USE_XMLRPC_CLIENT = os.environ.get("PUSHSOURCE_ERRATA_USE_XMLRPC_API") == "1"
 
+ERRATA_RETRY_STRATEGY = Retry(
+    total=5,
+    status_forcelist=range(500, 600),
+    backoff_factor=2,
+    allowed_methods=frozenset(["GET", "POST"]),
+    raise_on_status=False,
+)
+
 
 def get_errata_client(
     threads,
@@ -289,6 +299,9 @@ def _errata_service(self):
 
             session = requests.Session()
             session.auth = requests_gssapi.HTTPSPNEGOAuth(creds=creds)
+            adapter = requests.adapters.HTTPAdapter(max_retries=ERRATA_RETRY_STRATEGY)
+            session.mount("https://", adapter)
+            session.mount("http://", adapter)
             self._tls.session = session
 
         return self._tls.session

tests/errata/fake_errata_tool.py

@@ -4,6 +4,7 @@
 import yaml
 import json
 from xmlrpc.client import Fault  # nosec B411
+from urllib3.util.retry import Retry
 
 import requests
 
@@ -54,6 +55,7 @@ class FakeErrataToolProxy(object):
     def __init__(self, controller):
         self._ctrl = controller
         self.auth = None
+        self.mount = Mock(spec=Retry)
         self.url_map = [
             (".*/api/v1/erratum/(.*)", self.get_advisory_data),
             (

tests/errata/test_errata_client.py

@@ -7,6 +7,7 @@
 import requests
 
 from pushsource._impl.backend.errata_source.errata_client import (
+    ERRATA_RETRY_STRATEGY,
     ErrataHTTPClient,
     get_errata_client,
 )
@@ -35,11 +36,14 @@ def test_init_env_vars():
     assert client.principal == "[email protected]"
 
 
+@mock.patch("requests.adapters.HTTPAdapter")
 @mock.patch("gssapi.Name")
 @mock.patch("gssapi.Credentials.acquire")
 @mock.patch("requests.Session")
 @mock.patch("requests_gssapi.HTTPSPNEGOAuth")
-def test_get_session(mock_auth, mock_session, mock_acquire, mock_name, caplog):
+def test_get_session(
+    mock_auth, mock_session, mock_acquire, mock_name, mock_adapter, caplog
+):
     caplog.set_level(logging.DEBUG)
 
     client = ErrataHTTPClient(
@@ -57,6 +61,8 @@ def test_get_session(mock_auth, mock_session, mock_acquire, mock_name, caplog):
     )
     mock_session.assert_called_once_with()
     mock_auth.assert_called_once_with(creds=mock_acquire.return_value.creds)
+    mock_adapter.assert_called_once_with(max_retries=ERRATA_RETRY_STRATEGY)
+    assert session.mount.call_count == 2
 
     assert session == mock_session.return_value
     assert client._tls.session == mock_session.return_value