Add $feature_flag_error property to track flag evaluation failures

Track errors in feature flag evaluation by adding a `$feature_flag_error` property to the `$feature_flag_called` event.

Author: haacked

posthog/client.py

@@ -36,12 +36,14 @@
 from posthog.request import (
     DEFAULT_HOST,
     APIError,
+    QuotaLimitError,
     batch_post,
     determine_server_host,
     flags,
     get,
     remote_config,
 )
+import requests.exceptions
 from posthog.contexts import (
     _get_current_context,
     get_context_distinct_id,
@@ -1539,6 +1541,7 @@ def _get_feature_flag_result(
         flag_details = None
         request_id = None
         evaluated_at = None
+        feature_flag_error: Optional[str] = None
 
         flag_value = self._locally_evaluate_flag(
             key, distinct_id, groups, person_properties, group_properties
@@ -1563,7 +1566,7 @@ def _get_feature_flag_result(
                 )
         elif not only_evaluate_locally:
             try:
-                flag_details, request_id, evaluated_at = (
+                flag_details, request_id, evaluated_at, errors_while_computing = (
                     self._get_feature_flag_details_from_server(
                         key,
                         distinct_id,
@@ -1573,6 +1576,11 @@ def _get_feature_flag_result(
                         disable_geoip,
                     )
                 )
+                if errors_while_computing:
+                    feature_flag_error = "errors_while_computing_flags"
+                elif flag_details is None:
+                    feature_flag_error = "flag_missing"
+
                 flag_result = FeatureFlagResult.from_flag_details(
                     flag_details, override_match_value
                 )
@@ -1586,9 +1594,23 @@ def _get_feature_flag_result(
                 self.log.debug(
                     f"Successfully computed flag remotely: #{key} -> #{flag_result}"
                 )
+            except QuotaLimitError as e:
+                self.log.exception(f"[FEATURE FLAGS] Quota limit exceeded: {e}")
+                feature_flag_error = "quota_limited"
+            except requests.exceptions.Timeout as e:
+                self.log.exception(f"[FEATURE FLAGS] Request timed out: {e}")
+                feature_flag_error = "timeout"
+            except requests.exceptions.ConnectionError as e:
+                self.log.exception(f"[FEATURE FLAGS] Connection error: {e}")
+                feature_flag_error = "connection_error"
+            except APIError as e:
+                self.log.exception(f"[FEATURE FLAGS] API error: {e}")
+                feature_flag_error = f"api_error_{e.status}"
             except Exception as e:
                 self.log.exception(f"[FEATURE FLAGS] Unable to get flag remotely: {e}")
+                feature_flag_error = "unknown_error"
 
+            if feature_flag_error:
                 # Fallback to cached value if remote evaluation fails
                 if self.flag_cache:
                     stale_result = self.flag_cache.get_stale_cached_flag(
@@ -1612,6 +1634,7 @@ def _get_feature_flag_result(
                 request_id,
                 evaluated_at,
                 flag_details,
+                feature_flag_error,
             )
 
         return flag_result
@@ -1814,9 +1837,10 @@ def _get_feature_flag_details_from_server(
         person_properties: dict[str, str],
         group_properties: dict[str, str],
         disable_geoip: Optional[bool],
-    ) -> tuple[Optional[FeatureFlag], Optional[str], Optional[int]]:
+    ) -> tuple[Optional[FeatureFlag], Optional[str], Optional[int], bool]:
         """
-        Calls /flags and returns the flag details, request id, and evaluated at timestamp
+        Calls /flags and returns the flag details, request id, evaluated at timestamp,
+        and whether there were errors while computing flags.
         """
         resp_data = self.get_flags_decision(
             distinct_id,
@@ -1828,9 +1852,10 @@ def _get_feature_flag_details_from_server(
         )
         request_id = resp_data.get("requestId")
         evaluated_at = resp_data.get("evaluatedAt")
+        errors_while_computing = resp_data.get("errorsWhileComputingFlags", False)
         flags = resp_data.get("flags")
         flag_details = flags.get(key) if flags else None
-        return flag_details, request_id, evaluated_at
+        return flag_details, request_id, evaluated_at, errors_while_computing
 
     def _capture_feature_flag_called(
         self,
@@ -1844,6 +1869,7 @@ def _capture_feature_flag_called(
         request_id: Optional[str],
         evaluated_at: Optional[int],
         flag_details: Optional[FeatureFlag],
+        feature_flag_error: Optional[str] = None,
     ):
         feature_flag_reported_key = (
             f"{key}_{'::null::' if response is None else str(response)}"
@@ -1878,6 +1904,8 @@ def _capture_feature_flag_called(
                         )
                     if flag_details.metadata.id:
                         properties["$feature_flag_id"] = flag_details.metadata.id
+            if feature_flag_error:
+                properties["$feature_flag_error"] = feature_flag_error
 
             self.capture(
                 "$feature_flag_called",

posthog/test/test_feature_flag_result.py

@@ -438,6 +438,220 @@ def test_get_feature_flag_result_unknown_flag(self, patch_capture, patch_flags):
                 "$feature_flag_response": None,
                 "locally_evaluated": False,
                 "$feature/no-person-flag": None,
+                "$feature_flag_error": "flag_missing",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_with_errors_while_computing_flags(
+        self, patch_capture, patch_flags
+    ):
+        """Test that errors_while_computing_flags is included in the $feature_flag_called event.
+
+        When the server returns errorsWhileComputingFlags=true, it indicates that there
+        was an error computing one or more flags. We include this in the event so users
+        can identify and debug flag evaluation issues.
+        """
+        patch_flags.return_value = {
+            "flags": {
+                "my-flag": {
+                    "key": "my-flag",
+                    "enabled": True,
+                    "variant": None,
+                    "reason": {"description": "Matched condition set 1"},
+                    "metadata": {"id": 1, "version": 1, "payload": None},
+                },
+            },
+            "requestId": "test-request-id-789",
+            "errorsWhileComputingFlags": True,
+        }
+
+        flag_result = self.client.get_feature_flag_result("my-flag", "some-distinct-id")
+
+        self.assertEqual(flag_result.enabled, True)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "my-flag",
+                "$feature_flag_response": True,
+                "locally_evaluated": False,
+                "$feature/my-flag": True,
+                "$feature_flag_request_id": "test-request-id-789",
+                "$feature_flag_reason": "Matched condition set 1",
+                "$feature_flag_id": 1,
+                "$feature_flag_version": 1,
+                "$feature_flag_error": "errors_while_computing_flags",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_flag_not_in_response(
+        self, patch_capture, patch_flags
+    ):
+        """Test that when a flag is not in the API response, we capture flag_missing error.
+
+        This happens when a flag doesn't exist or the user doesn't match any conditions.
+        """
+        patch_flags.return_value = {
+            "flags": {
+                "other-flag": {
+                    "key": "other-flag",
+                    "enabled": True,
+                    "variant": None,
+                    "reason": {"description": "Matched condition set 1"},
+                    "metadata": {"id": 1, "version": 1, "payload": None},
+                },
+            },
+            "requestId": "test-request-id-456",
+        }
+
+        flag_result = self.client.get_feature_flag_result(
+            "missing-flag", "some-distinct-id"
+        )
+
+        self.assertIsNone(flag_result)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "missing-flag",
+                "$feature_flag_response": None,
+                "locally_evaluated": False,
+                "$feature/missing-flag": None,
+                "$feature_flag_request_id": "test-request-id-456",
+                "$feature_flag_error": "flag_missing",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_unknown_error(self, patch_capture, patch_flags):
+        """Test that unexpected exceptions are captured as unknown_error."""
+        patch_flags.side_effect = Exception("Unexpected error")
+
+        flag_result = self.client.get_feature_flag_result("my-flag", "some-distinct-id")
+
+        self.assertIsNone(flag_result)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "my-flag",
+                "$feature_flag_response": None,
+                "locally_evaluated": False,
+                "$feature/my-flag": None,
+                "$feature_flag_error": "unknown_error",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_timeout_error(self, patch_capture, patch_flags):
+        """Test that timeout errors are captured specifically."""
+        import requests.exceptions
+
+        patch_flags.side_effect = requests.exceptions.Timeout("Request timed out")
+
+        flag_result = self.client.get_feature_flag_result("my-flag", "some-distinct-id")
+
+        self.assertIsNone(flag_result)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "my-flag",
+                "$feature_flag_response": None,
+                "locally_evaluated": False,
+                "$feature/my-flag": None,
+                "$feature_flag_error": "timeout",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_connection_error(self, patch_capture, patch_flags):
+        """Test that connection errors are captured specifically."""
+        import requests.exceptions
+
+        patch_flags.side_effect = requests.exceptions.ConnectionError(
+            "Connection refused"
+        )
+
+        flag_result = self.client.get_feature_flag_result("my-flag", "some-distinct-id")
+
+        self.assertIsNone(flag_result)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "my-flag",
+                "$feature_flag_response": None,
+                "locally_evaluated": False,
+                "$feature/my-flag": None,
+                "$feature_flag_error": "connection_error",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_api_error(self, patch_capture, patch_flags):
+        """Test that API errors include the status code."""
+        from posthog.request import APIError
+
+        patch_flags.side_effect = APIError(500, "Internal server error")
+
+        flag_result = self.client.get_feature_flag_result("my-flag", "some-distinct-id")
+
+        self.assertIsNone(flag_result)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "my-flag",
+                "$feature_flag_response": None,
+                "locally_evaluated": False,
+                "$feature/my-flag": None,
+                "$feature_flag_error": "api_error_500",
+            },
+            groups={},
+            disable_geoip=None,
+        )
+
+    @mock.patch("posthog.client.flags")
+    @mock.patch.object(Client, "capture")
+    def test_get_feature_flag_result_quota_limited(self, patch_capture, patch_flags):
+        """Test that quota limit errors are captured specifically."""
+        from posthog.request import QuotaLimitError
+
+        patch_flags.side_effect = QuotaLimitError(429, "Rate limit exceeded")
+
+        flag_result = self.client.get_feature_flag_result("my-flag", "some-distinct-id")
+
+        self.assertIsNone(flag_result)
+        patch_capture.assert_called_with(
+            "$feature_flag_called",
+            distinct_id="some-distinct-id",
+            properties={
+                "$feature_flag": "my-flag",
+                "$feature_flag_response": None,
+                "locally_evaluated": False,
+                "$feature/my-flag": None,
+                "$feature_flag_error": "quota_limited",
             },
             groups={},
             disable_geoip=None,