Add retry logic with jitter for feature flag requests

Add exponential backoff with full jitter for feature flag API calls to handle transient failures like timeouts and connection errors.

- Default 2 retries (3 total attempts) with exponential backoff
- Full jitter to prevent thundering herd during incidents
- Retries on: timeouts, connection errors, 5xx, 429
- No retry on: 4xx client errors, quota limit errors
- Configurable via `feature_flag_retries` parameter on Client

Author: haacked

posthog/client.py

@@ -34,6 +34,7 @@
 )
 from posthog.poller import Poller
 from posthog.request import (
+    DEFAULT_FEATURE_FLAG_RETRIES,
     DEFAULT_HOST,
     APIError,
     QuotaLimitError,
@@ -196,6 +197,7 @@ def __init__(
         capture_exception_code_variables=False,
         code_variables_mask_patterns=None,
         code_variables_ignore_patterns=None,
+        feature_flag_retries=DEFAULT_FEATURE_FLAG_RETRIES,
     ):
         """
         Initialize a new PostHog client instance.
@@ -237,6 +239,7 @@ def __init__(
         self.feature_flags_request_timeout_seconds = (
             feature_flags_request_timeout_seconds
         )
+        self.feature_flag_retries = feature_flag_retries
         self.poller = None
         self.distinct_ids_feature_flags_reported = SizeLimitedDict(MAX_DICT_SIZE, set)
         self.flag_cache = self._initialize_flag_cache(flag_fallback_cache_url)
@@ -540,6 +543,7 @@ def get_flags_decision(
             self.api_key,
             self.host,
             timeout=self.feature_flags_request_timeout_seconds,
+            retries=self.feature_flag_retries,
             **request_data,
         )
 

posthog/request.py

@@ -6,9 +6,9 @@
 from datetime import date, datetime
 from gzip import GzipFile
 from io import BytesIO
-from typing import Any, List, Optional, Tuple, Union
-
+from typing import Any, Callable, List, Optional, Tuple, TypeVar, Union
 
+import backoff
 import requests
 from dateutil.tz import tzutc
 from requests.adapters import HTTPAdapter  # type: ignore[import-untyped]
@@ -42,6 +42,83 @@
     if hasattr(socket, attr):
         KEEP_ALIVE_SOCKET_OPTIONS.append((socket.SOL_TCP, getattr(socket, attr), value))
 
+# Default number of retries for feature flag requests
+DEFAULT_FEATURE_FLAG_RETRIES = 2
+
+T = TypeVar("T")
+
+
+def _should_giveup_on_exception(exc: Exception) -> bool:
+    """
+    Determine if we should stop retrying based on the exception type.
+
+    Returns True to give up (stop retrying), False to continue retrying.
+    """
+    # Note: APIError and QuotaLimitError are defined later in this module.
+    # We use late binding here since this function is defined before those classes.
+    if isinstance(exc, QuotaLimitError):
+        # Don't retry quota limit errors - they won't resolve with retries
+        return True
+    if isinstance(exc, APIError):
+        # Retry on server errors (5xx) and rate limits (429)
+        # Don't retry on other client errors (4xx)
+        if not isinstance(exc.status, int):
+            return False
+        return (400 <= exc.status < 500) and exc.status != 429
+    # Retry on all other errors (network errors, timeouts, etc.)
+    return False
+
+
+def _with_retries(
+    fn: Callable[[], T],
+    retries: int,
+    giveup: Callable[[Exception], bool] = _should_giveup_on_exception,
+) -> T:
+    """
+    Execute a function with exponential backoff retries and jitter.
+
+    Uses exponential backoff with full jitter to spread out retry attempts
+    and prevent thundering herd problems during incidents.
+
+    Note: This application-level retry operates independently from urllib3's
+    transport-level retries (configured in HTTPAdapterWithSocketOptions).
+    With default settings, each backoff attempt may trigger up to 4 urllib3
+    retries (2 connect + 2 read), so total network attempts could be higher
+    than the retry count suggests.
+
+    Args:
+        fn: The function to execute
+        retries: Maximum number of retries (0 means no retries, just one attempt)
+        giveup: Function that returns True if we should stop retrying for a given exception
+
+    Returns:
+        The result of the function call
+    """
+    log = logging.getLogger("posthog")
+
+    def on_backoff(details):
+        log.warning(
+            "Request failed (attempt %d/%d), retrying in %.2fs: %s",
+            details["tries"],
+            retries + 1,
+            details["wait"],
+            details["exception"],
+        )
+
+    # max_tries = retries + 1 (e.g., retries=2 means 3 total attempts)
+    @backoff.on_exception(
+        backoff.expo,
+        Exception,
+        max_tries=retries + 1,
+        giveup=giveup,
+        jitter=backoff.full_jitter,  # Adds randomness to prevent thundering herd
+        on_backoff=on_backoff,
+    )
+    def execute():
+        return fn()
+
+    return execute()
+
 
 def _mask_tokens_in_url(url: str) -> str:
     """Mask token values in URLs for safe logging, keeping first 10 chars visible."""
@@ -219,13 +296,21 @@ def flags(
     host: Optional[str] = None,
     gzip: bool = False,
     timeout: int = 15,
+    retries: int = DEFAULT_FEATURE_FLAG_RETRIES,
     **kwargs,
 ) -> Any:
-    """Post the `kwargs to the flags API endpoint"""
-    res = post(api_key, host, "/flags/?v=2", gzip, timeout, **kwargs)
-    return _process_response(
-        res, success_message="Feature flags evaluated successfully"
-    )
+    """Post the `kwargs to the flags API endpoint with automatic retries."""
+
+    def make_request():
+        res = post(api_key, host, "/flags/?v=2", gzip, timeout, **kwargs)
+        return _process_response(
+            res, success_message="Feature flags evaluated successfully"
+        )
+
+    if retries > 0:
+        return _with_retries(make_request, retries)
+    else:
+        return make_request()
 
 
 def remote_config(

posthog/test/test_client.py

@@ -9,7 +9,7 @@
 
 from posthog.client import Client
 from posthog.contexts import get_context_session_id, new_context, set_context_session
-from posthog.request import APIError, GetResponse
+from posthog.request import APIError, DEFAULT_FEATURE_FLAG_RETRIES, GetResponse
 from posthog.test.test_utils import FAKE_TEST_API_KEY
 from posthog.types import FeatureFlag, LegacyFlagMetadata
 from posthog.version import VERSION
@@ -640,6 +640,7 @@ def test_basic_capture_with_feature_flags_returns_active_only(self, patch_flags)
                 "random_key",
                 "https://us.i.posthog.com",
                 timeout=3,
+                retries=DEFAULT_FEATURE_FLAG_RETRIES,
                 distinct_id="distinct_id",
                 groups={},
                 person_properties={},
@@ -704,6 +705,7 @@ def test_basic_capture_with_feature_flags_and_disable_geoip_returns_correctly(
                 "random_key",
                 "https://us.i.posthog.com",
                 timeout=12,
+                retries=DEFAULT_FEATURE_FLAG_RETRIES,
                 distinct_id="distinct_id",
                 groups={},
                 person_properties={},
@@ -1821,6 +1823,7 @@ def test_disable_geoip_default_on_decide(self, patch_flags):
             "random_key",
             "https://us.i.posthog.com",
             timeout=3,
+            retries=DEFAULT_FEATURE_FLAG_RETRIES,
             distinct_id="some_id",
             groups={},
             person_properties={"distinct_id": "some_id"},
@@ -1836,6 +1839,7 @@ def test_disable_geoip_default_on_decide(self, patch_flags):
             "random_key",
             "https://us.i.posthog.com",
             timeout=3,
+            retries=DEFAULT_FEATURE_FLAG_RETRIES,
             distinct_id="feature_enabled_distinct_id",
             groups={},
             person_properties={"distinct_id": "feature_enabled_distinct_id"},
@@ -1849,6 +1853,7 @@ def test_disable_geoip_default_on_decide(self, patch_flags):
             "random_key",
             "https://us.i.posthog.com",
             timeout=3,
+            retries=DEFAULT_FEATURE_FLAG_RETRIES,
             distinct_id="all_flags_payloads_id",
             groups={},
             person_properties={"distinct_id": "all_flags_payloads_id"},
@@ -1894,6 +1899,7 @@ def test_default_properties_get_added_properly(self, patch_flags):
             "random_key",
             "http://app2.posthog.com",
             timeout=3,
+            retries=DEFAULT_FEATURE_FLAG_RETRIES,
             distinct_id="some_id",
             groups={"company": "id:5", "instance": "app.posthog.com"},
             person_properties={"distinct_id": "some_id", "x1": "y1"},
@@ -1921,6 +1927,7 @@ def test_default_properties_get_added_properly(self, patch_flags):
             "random_key",
             "http://app2.posthog.com",
             timeout=3,
+            retries=DEFAULT_FEATURE_FLAG_RETRIES,
             distinct_id="some_id",
             groups={"company": "id:5", "instance": "app.posthog.com"},
             person_properties={"distinct_id": "override"},
@@ -1941,13 +1948,39 @@ def test_default_properties_get_added_properly(self, patch_flags):
             "random_key",
             "http://app2.posthog.com",
             timeout=3,
+            retries=DEFAULT_FEATURE_FLAG_RETRIES,
             distinct_id="some_id",
             groups={},
             person_properties={"distinct_id": "some_id"},
             group_properties={},
             geoip_disable=False,
         )
 
+    @mock.patch("posthog.client.flags")
+    def test_client_passes_custom_retry_count_to_flags(self, patch_flags):
+        """Verify Client passes feature_flag_retries parameter to flags()."""
+        patch_flags.return_value = {
+            "featureFlags": {"test-flag": True},
+            "featureFlagPayloads": {},
+            "errorsWhileComputingFlags": False,
+        }
+
+        # Create client with custom retry count
+        client = Client(
+            FAKE_TEST_API_KEY,
+            host="https://test.posthog.com",
+            feature_flag_retries=5,
+            on_error=self.set_fail,
+        )
+
+        # Trigger flag evaluation
+        client.get_feature_flag("test-flag", "user123")
+
+        # Verify retries=5 was passed to flags()
+        patch_flags.assert_called_once()
+        call_kwargs = patch_flags.call_args[1]
+        self.assertEqual(call_kwargs["retries"], 5)
+
     @parameterized.expand(
         [
             # name, sys_platform, version_info, expected_runtime, expected_version, expected_os, expected_os_version, platform_method, platform_return, distro_info