Retry Jira and Bugzilla on error (fixes #33) (#152)

* Fixup #134: fix decorated client methods

* Retry Jira and Bugzilla on error (fixes #33)

Author: leplatrem

poetry.lock

@@ -18,6 +18,7 @@ dockerflow = "2022.7.0"
 Jinja2 = "^3.1.2"
 pydantic-yaml = {extras = ["pyyaml","ruamel"], version = "^0.6.1"}
 sentry-sdk = "^1.8.0"
+backoff = "^2.1.2"
 
 
 [tool.poetry.dev-dependencies]

pyproject.toml

@@ -20,6 +20,7 @@ class Settings(BaseSettings):
     port: int = 8000
     app_reload: bool = True
     env: str = "nonprod"
+    max_retries: int = 3
 
     # Jira
     jira_base_url: str = "https://mozit-test.atlassian.net/"

src/app/environment.py

@@ -30,6 +30,7 @@ def configure_logging():
         },
         "loggers": {
             "request.summary": {"handlers": ["console"], "level": "INFO"},
+            "backoff": {"handlers": ["console"], "level": "INFO"},
             "src.jbi": {"handlers": ["console"], "level": "DEBUG"},
         },
     }

src/app/log.py

@@ -1,10 +1,10 @@
 """Services and functions that can be used to create custom actions"""
-import contextlib
 import logging
 from typing import Dict, List
 
+import backoff
 import bugzilla as rh_bugzilla
-from atlassian import Jira
+from atlassian import Jira, errors
 from prometheus_client import Counter, Summary
 
 from src.app import environment
@@ -19,16 +19,18 @@
 
 
 class InstrumentedClient:
-    """This class wraps an object and increments a counter everytime
-    the specieds methods are called, and times their execution.
+    """This class wraps an object and increments a counter every time
+    the specified methods are called, and times their execution.
+    It retries the methods if the specified exceptions are raised.
     """
 
     counters: Dict[str, Counter] = {}
     timers: Dict[str, Counter] = {}
 
-    def __init__(self, wrapped, prefix, methods):
+    def __init__(self, wrapped, prefix, methods, exceptions):
         self.wrapped = wrapped
         self.methods = methods
+        self.exceptions = exceptions
 
         # We have a single instrument per prefix. Methods are reported as labels.
         counter_name = prefix + "_methods_total"
@@ -46,14 +48,23 @@ def __init__(self, wrapped, prefix, methods):
         self.timer = self.timers[timer_name]
 
     def __getattr__(self, attr):
-        if attr in self.methods:
+        if attr not in self.methods:
+            return getattr(self.wrapped, attr)
+
+        @backoff.on_exception(
+            backoff.expo,
+            self.exceptions,
+            max_tries=settings.max_retries + 1,
+        )
+        def wrapped_func(*args, **kwargs):
+            # Increment the call counter.
             self.counter.labels(method=attr).inc()
-            timer_cm = self.timer.labels(method=attr).time()
-        else:
-            timer_cm = contextlib.nullcontext()
+            # Time its execution.
+            with self.timer.labels(method=attr).time():
+                return getattr(self.wrapped, attr)(*args, **kwargs)
 
-        with timer_cm:
-            return getattr(self.wrapped, attr)
+        # The method was not called yet.
+        return wrapped_func
 
 
 def get_jira():
@@ -71,7 +82,10 @@ def get_jira():
         "create_issue",
     )
     return InstrumentedClient(
-        wrapped=jira_client, prefix="jbi_jira", methods=instrumented_methods
+        wrapped=jira_client,
+        prefix="jbi_jira",
+        methods=instrumented_methods,
+        exceptions=(errors.ApiError,),
     )
 
 
@@ -93,7 +107,10 @@ def get_bugzilla():
         "update_bugs",
     )
     return InstrumentedClient(
-        wrapped=bugzilla_client, prefix="jbi_bugzilla", methods=instrumented_methods
+        wrapped=bugzilla_client,
+        prefix="jbi_bugzilla",
+        methods=instrumented_methods,
+        exceptions=(rh_bugzilla.BugzillaError,),
     )
 
 

src/jbi/services.py

@@ -3,6 +3,7 @@
 """
 from unittest import mock
 
+import bugzilla
 import pytest
 
 from src.jbi.services import get_bugzilla, get_jira
@@ -45,3 +46,16 @@ def test_timer_is_used_on_bugzilla_getcomments():
             bugzilla_client.get_comments([])
 
     assert mocked.called, "Timer was used on get_comments()"
+
+
+def test_bugzilla_methods_are_retried_if_raising():
+    with mock.patch(
+        "src.jbi.services.rh_bugzilla.Bugzilla.return_value.get_comments"
+    ) as mocked:
+        mocked.side_effect = (bugzilla.BugzillaError("boom"), [mock.sentinel])
+        bugzilla_client = get_bugzilla()
+
+        # Not raising
+        bugzilla_client.get_comments([])
+
+    assert mocked.call_count == 2