feature: Add $feature_flag_error property to track flag evaluation failures (#390)

* Add $feature_flag_error property to track flag evaluation failures

Track errors in feature flag evaluation by adding a `$feature_flag_error` property to the `$feature_flag_called` event.

* Refactor requests exception imports through request.py

Export RequestsTimeout and RequestsConnectionError from posthog/request.py
to keep all requests library imports in one place and avoid mypy issues.

* Address PR review feedback

- Fix fallback logic to only trigger on actual exceptions, not when
  errors_while_computing or flag_missing is set from a successful API response
- Change log.exception() to log.warning() for expected operational errors
  (quota limits, timeouts, connection errors, API errors) to reduce log noise
- Keep log.exception() only for truly unexpected errors (unknown_error)
- Extract stale cache fallback into _get_stale_flag_fallback() helper method

* Add tests for stale cache fallback and error absence

- Add TestFeatureFlagErrorWithStaleCacheFallback test class with 4 tests:
  - test_timeout_error_returns_stale_cached_value
  - test_connection_error_returns_stale_cached_value
  - test_api_error_returns_stale_cached_value
  - test_error_without_cache_returns_none

- Add negative assertions to verify $feature_flag_error is absent on success:
  - test_get_feature_flag_result_boolean_local_evaluation
  - test_get_feature_flag_result_variant_local_evaluation
  - test_get_feature_flag_result_boolean_decide
  - test_get_feature_flag_result_variant_decide

* Report combined errors when both errors_while_computing and flag_missing

When the server returns errorsWhileComputingFlags=true AND the requested
flag is not in the response, report both conditions as a comma-separated
string: "errors_while_computing_flags,flag_missing"

This provides better debugging context when both conditions occur.

* Add FeatureFlagError constants class for error type values

- Add FeatureFlagError class to types.py with constants:
  - ERRORS_WHILE_COMPUTING, FLAG_MISSING, QUOTA_LIMITED
  - TIMEOUT, CONNECTION_ERROR, UNKNOWN_ERROR
  - api_error(status) static method for dynamic error strings

- Update client.py to use FeatureFlagError constants instead of
  magic strings

- Update all tests to use constants for maintainability

This improves maintainability by:
- Single source of truth for error values
- IDE autocomplete and typo detection
- Documentation of analytics-stable values

* Remove print statements from test failure handlers

* Fix mypy type error in FeatureFlagError.api_error method

Accept Union[int, str] to match APIError.status type.

Author: haacked

posthog/client.py

@@ -36,6 +36,9 @@
 from posthog.request import (
     DEFAULT_HOST,
     APIError,
+    QuotaLimitError,
+    RequestsConnectionError,
+    RequestsTimeout,
     batch_post,
     determine_server_host,
     flags,
@@ -53,6 +56,7 @@
 )
 from posthog.types import (
     FeatureFlag,
+    FeatureFlagError,
     FeatureFlagResult,
     FlagMetadata,
     FlagsAndPayloads,
@@ -1506,6 +1510,19 @@ def feature_enabled(
             return None
         return bool(response)
 
+    def _get_stale_flag_fallback(
+        self, distinct_id: ID_TYPES, key: str
+    ) -> Optional[FeatureFlagResult]:
+        """Returns a stale cached flag value if available, otherwise None."""
+        if self.flag_cache:
+            stale_result = self.flag_cache.get_stale_cached_flag(distinct_id, key)
+            if stale_result:
+                self.log.info(
+                    f"[FEATURE FLAGS] Using stale cached value for flag {key}"
+                )
+                return stale_result
+        return None
+
     def _get_feature_flag_result(
         self,
         key: str,
@@ -1539,6 +1556,7 @@ def _get_feature_flag_result(
         flag_details = None
         request_id = None
         evaluated_at = None
+        feature_flag_error: Optional[str] = None
 
         flag_value = self._locally_evaluate_flag(
             key, distinct_id, groups, person_properties, group_properties
@@ -1563,7 +1581,7 @@ def _get_feature_flag_result(
                 )
         elif not only_evaluate_locally:
             try:
-                flag_details, request_id, evaluated_at = (
+                flag_details, request_id, evaluated_at, errors_while_computing = (
                     self._get_feature_flag_details_from_server(
                         key,
                         distinct_id,
@@ -1573,6 +1591,14 @@ def _get_feature_flag_result(
                         disable_geoip,
                     )
                 )
+                errors = []
+                if errors_while_computing:
+                    errors.append(FeatureFlagError.ERRORS_WHILE_COMPUTING)
+                if flag_details is None:
+                    errors.append(FeatureFlagError.FLAG_MISSING)
+                if errors:
+                    feature_flag_error = ",".join(errors)
+
                 flag_result = FeatureFlagResult.from_flag_details(
                     flag_details, override_match_value
                 )
@@ -1586,19 +1612,26 @@ def _get_feature_flag_result(
                 self.log.debug(
                     f"Successfully computed flag remotely: #{key} -> #{flag_result}"
                 )
+            except QuotaLimitError as e:
+                self.log.warning(f"[FEATURE FLAGS] Quota limit exceeded: {e}")
+                feature_flag_error = FeatureFlagError.QUOTA_LIMITED
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
+            except RequestsTimeout as e:
+                self.log.warning(f"[FEATURE FLAGS] Request timed out: {e}")
+                feature_flag_error = FeatureFlagError.TIMEOUT
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
+            except RequestsConnectionError as e:
+                self.log.warning(f"[FEATURE FLAGS] Connection error: {e}")
+                feature_flag_error = FeatureFlagError.CONNECTION_ERROR
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
+            except APIError as e:
+                self.log.warning(f"[FEATURE FLAGS] API error: {e}")
+                feature_flag_error = FeatureFlagError.api_error(e.status)
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
             except Exception as e:
                 self.log.exception(f"[FEATURE FLAGS] Unable to get flag remotely: {e}")
-
-                # Fallback to cached value if remote evaluation fails
-                if self.flag_cache:
-                    stale_result = self.flag_cache.get_stale_cached_flag(
-                        distinct_id, key
-                    )
-                    if stale_result:
-                        self.log.info(
-                            f"[FEATURE FLAGS] Using stale cached value for flag {key}"
-                        )
-                        flag_result = stale_result
+                feature_flag_error = FeatureFlagError.UNKNOWN_ERROR
+                flag_result = self._get_stale_flag_fallback(distinct_id, key)
 
         if send_feature_flag_events:
             self._capture_feature_flag_called(
@@ -1612,6 +1645,7 @@ def _get_feature_flag_result(
                 request_id,
                 evaluated_at,
                 flag_details,
+                feature_flag_error,
             )
 
         return flag_result
@@ -1814,9 +1848,10 @@ def _get_feature_flag_details_from_server(
         person_properties: dict[str, str],
         group_properties: dict[str, str],
         disable_geoip: Optional[bool],
-    ) -> tuple[Optional[FeatureFlag], Optional[str], Optional[int]]:
+    ) -> tuple[Optional[FeatureFlag], Optional[str], Optional[int], bool]:
         """
-        Calls /flags and returns the flag details, request id, and evaluated at timestamp
+        Calls /flags and returns the flag details, request id, evaluated at timestamp,
+        and whether there were errors while computing flags.
         """
         resp_data = self.get_flags_decision(
             distinct_id,
@@ -1828,9 +1863,10 @@ def _get_feature_flag_details_from_server(
         )
         request_id = resp_data.get("requestId")
         evaluated_at = resp_data.get("evaluatedAt")
+        errors_while_computing = resp_data.get("errorsWhileComputingFlags", False)
         flags = resp_data.get("flags")
         flag_details = flags.get(key) if flags else None
-        return flag_details, request_id, evaluated_at
+        return flag_details, request_id, evaluated_at, errors_while_computing
 
     def _capture_feature_flag_called(
         self,
@@ -1844,6 +1880,7 @@ def _capture_feature_flag_called(
         request_id: Optional[str],
         evaluated_at: Optional[int],
         flag_details: Optional[FeatureFlag],
+        feature_flag_error: Optional[str] = None,
     ):
         feature_flag_reported_key = (
             f"{key}_{'::null::' if response is None else str(response)}"
@@ -1878,6 +1915,8 @@ def _capture_feature_flag_called(
                         )
                     if flag_details.metadata.id:
                         properties["$feature_flag_id"] = flag_details.metadata.id
+            if feature_flag_error:
+                properties["$feature_flag_error"] = feature_flag_error
 
             self.capture(
                 "$feature_flag_called",

posthog/request.py

@@ -312,6 +312,12 @@ class QuotaLimitError(APIError):
     pass
 
 
+# Re-export requests exceptions for use in client.py
+# This keeps all requests library imports centralized in this module
+RequestsTimeout = requests.exceptions.Timeout
+RequestsConnectionError = requests.exceptions.ConnectionError
+
+
 class DatetimeSerializer(json.JSONEncoder):
     def default(self, obj: Any):
         if isinstance(obj, (date, datetime)):