Merge pull request #67 from jschlyter/key_bundle_backoff

Ignore KeyBundle errors for a specified duration

When fetching remote keys, KeyBundle will retry every time even if errors had recently occurred. This might not be desirable for many applications, as errors like this usually fix themselves within a short amount of time. Until then, additional requests would result in more load, latency, excess logging and key source spamming.

The PR adds an `ignore_errors_period` parameter to the KeyBundle object, that is used to specify the amount of time (in seconds) during which fetching is skipped. Default value is `0` (zero), effectively disabling this functionality and resulting in retries every time the bundle is accessed. For most applications, a hold down timer of 60 seconds or more is probably reasonable.

Author: c00kiemon5ter

src/cryptojwt/key_bundle.py

@@ -4,6 +4,7 @@
 import logging
 import os
 import time
+from datetime import datetime
 from functools import cmp_to_key
 
 import requests
@@ -156,6 +157,7 @@ def __init__(
         keys=None,
         source="",
         cache_time=300,
+        ignore_errors_period=0,
         fileformat="jwks",
         keytype="RSA",
         keyusage=None,
@@ -188,6 +190,8 @@ def __init__(
         self.remote = False
         self.local = False
         self.cache_time = cache_time
+        self.ignore_errors_period = ignore_errors_period
+        self.ignore_errors_until = None  # UNIX timestamp of last error
         self.time_out = 0
         self.etag = ""
         self.source = None
@@ -365,6 +369,14 @@ def do_remote(self):
         # if self.verify_ssl is not None:
         #     self.httpc_params["verify"] = self.verify_ssl
 
+        if self.ignore_errors_until and time.time() < self.ignore_errors_until:
+            LOGGER.warning(
+                "Not reading remote JWKS from %s (in error holddown until %s)",
+                self.source,
+                datetime.fromtimestamp(self.ignore_errors_until),
+            )
+            return False
+
         LOGGER.info("Reading remote JWKS from %s", self.source)
         try:
             LOGGER.debug("KeyBundle fetch keys from: %s", self.source)
@@ -390,6 +402,7 @@ def do_remote(self):
                 self.do_keys(self.imp_jwks["keys"])
             except KeyError:
                 LOGGER.error("No 'keys' keyword in JWKS")
+                self.ignore_errors_until = time.time() + self.ignore_errors_period
                 raise UpdateFailed(MALFORMED.format(self.source))
 
             if hasattr(_http_resp, "headers"):
@@ -406,8 +419,11 @@ def do_remote(self):
                 _http_resp.status_code,
                 self.source,
             )
+            self.ignore_errors_until = time.time() + self.ignore_errors_period
             raise UpdateFailed(REMOTE_FAILED.format(self.source, _http_resp.status_code))
+
         self.last_updated = time.time()
+        self.ignore_errors_until = None
         return True
 
     def _parse_remote_response(self, response):

tests/test_03_key_bundle.py

@@ -17,6 +17,7 @@
 from cryptojwt.jwk.rsa import import_rsa_key_from_cert_file
 from cryptojwt.jwk.rsa import new_rsa_key
 from cryptojwt.key_bundle import KeyBundle
+from cryptojwt.key_bundle import UpdateFailed
 from cryptojwt.key_bundle import build_key_bundle
 from cryptojwt.key_bundle import dump_jwks
 from cryptojwt.key_bundle import init_key
@@ -1024,3 +1025,43 @@ def test_remote_not_modified():
     assert kb2.httpc_params == {"timeout": (2, 2)}
     assert kb2.imp_jwks
     assert kb2.last_updated
+
+
+def test_ignore_errors_period():
+    source_good = "https://example.com/keys.json"
+    source_bad = "https://example.com/keys-bad.json"
+    ignore_errors_period = 1
+    # Mock response
+    with responses.RequestsMock() as rsps:
+        rsps.add(method="GET", url=source_good, json=JWKS_DICT, status=200)
+        rsps.add(method="GET", url=source_bad, json=JWKS_DICT, status=500)
+        httpc_params = {"timeout": (2, 2)}  # connect, read timeouts in seconds
+        kb = KeyBundle(
+            source=source_good,
+            httpc=requests.request,
+            httpc_params=httpc_params,
+            ignore_errors_period=ignore_errors_period,
+        )
+        res = kb.do_remote()
+        assert res == True
+        assert kb.ignore_errors_until is None
+
+        # refetch, but fail by using a bad source
+        kb.source = source_bad
+        try:
+            res = kb.do_remote()
+        except UpdateFailed:
+            pass
+
+        # retry should fail silently as we're in holddown
+        res = kb.do_remote()
+        assert kb.ignore_errors_until is not None
+        assert res == False
+
+        # wait until holddown
+        time.sleep(ignore_errors_period + 1)
+
+        # try again
+        kb.source = source_good
+        res = kb.do_remote()
+        assert res == True