feat(flags): Add integration for custom tracking of flag evaluations (#3860)

* Add new integration and unit tests

* Test flag values for LD and OF threaded/asyncio, not just flag names

* update ffIntegration test to be e2e, and fix LRU copy bug

* make a helper fixture and test error processor in original thread

* Move api to top-level, rename to add_flag

* Add docstrs

* Rename to add_feature_flag

* Rm extra import in test_lru_cache

* Revert lru comment

* Type annotate

* Review comments

* Update launchdarkly and openfeature tests to be e2e

* Update docstrs

* Skip threading test for <3.7

* Skip ffs asyncio test if 3.6

* undo 'skip threading test'

* Try commenting out asyncio

* Use importorskip

* Import order

---------

Co-authored-by: Anton Pirker <[email protected]>

Author: aliu39

sentry_sdk/integrations/featureflags.py

@@ -0,0 +1,44 @@
+from sentry_sdk.flag_utils import flag_error_processor
+
+import sentry_sdk
+from sentry_sdk.integrations import Integration
+
+
+class FeatureFlagsIntegration(Integration):
+    """
+    Sentry integration for capturing feature flags on error events. To manually buffer flag data,
+    call `integrations.featureflags.add_feature_flag`. We recommend you do this on each flag
+    evaluation.
+
+    See the [feature flag documentation](https://develop.sentry.dev/sdk/expected-features/#feature-flags)
+    for more information.
+
+    @example
+    ```
+    import sentry_sdk
+    from sentry_sdk.integrations.featureflags import FeatureFlagsIntegration, add_feature_flag
+
+    sentry_sdk.init(dsn="my_dsn", integrations=[FeatureFlagsIntegration()]);
+
+    add_feature_flag('my-flag', true);
+    sentry_sdk.capture_exception(Exception('broke')); // 'my-flag' should be captured on this Sentry event.
+    ```
+    """
+
+    identifier = "featureflags"
+
+    @staticmethod
+    def setup_once():
+        # type: () -> None
+        scope = sentry_sdk.get_current_scope()
+        scope.add_error_processor(flag_error_processor)
+
+
+def add_feature_flag(flag, result):
+    # type: (str, bool) -> None
+    """
+    Records a flag and its value to be sent on subsequent error events by FeatureFlagsIntegration.
+    We recommend you do this on flag evaluations. Flags are buffered per Sentry scope.
+    """
+    flags = sentry_sdk.get_current_scope().flags
+    flags.set(flag, result)

tests/conftest.py

@@ -184,6 +184,17 @@ def reset_integrations():
     _installed_integrations.clear()
 
 
+@pytest.fixture
+def uninstall_integration():
+    """Use to force the next call to sentry_init to re-install/setup an integration."""
+
+    def inner(identifier):
+        _processed_integrations.discard(identifier)
+        _installed_integrations.discard(identifier)
+
+    return inner
+
+
 @pytest.fixture
 def sentry_init(request):
     def inner(*a, **kw):

tests/integrations/featureflags/__init__.py

@@ -0,0 +1,133 @@
+import concurrent.futures as cf
+import sys
+
+import pytest
+
+import sentry_sdk
+from sentry_sdk.integrations.featureflags import (
+    FeatureFlagsIntegration,
+    add_feature_flag,
+)
+
+
+def test_featureflags_integration(sentry_init, capture_events, uninstall_integration):
+    uninstall_integration(FeatureFlagsIntegration.identifier)
+    sentry_init(integrations=[FeatureFlagsIntegration()])
+
+    add_feature_flag("hello", False)
+    add_feature_flag("world", True)
+    add_feature_flag("other", False)
+
+    events = capture_events()
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 1
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "world", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
+
+
+def test_featureflags_integration_threaded(
+    sentry_init, capture_events, uninstall_integration
+):
+    uninstall_integration(FeatureFlagsIntegration.identifier)
+    sentry_init(integrations=[FeatureFlagsIntegration()])
+    events = capture_events()
+
+    # Capture an eval before we split isolation scopes.
+    add_feature_flag("hello", False)
+
+    def task(flag_key):
+        # Creates a new isolation scope for the thread.
+        # This means the evaluations in each task are captured separately.
+        with sentry_sdk.isolation_scope():
+            add_feature_flag(flag_key, False)
+            # use a tag to identify to identify events later on
+            sentry_sdk.set_tag("task_id", flag_key)
+            sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    # Run tasks in separate threads
+    with cf.ThreadPoolExecutor(max_workers=2) as pool:
+        pool.map(task, ["world", "other"])
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "world", "result": False},
+        ]
+    }
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher")
+def test_featureflags_integration_asyncio(
+    sentry_init, capture_events, uninstall_integration
+):
+    asyncio = pytest.importorskip("asyncio")
+
+    uninstall_integration(FeatureFlagsIntegration.identifier)
+    sentry_init(integrations=[FeatureFlagsIntegration()])
+    events = capture_events()
+
+    # Capture an eval before we split isolation scopes.
+    add_feature_flag("hello", False)
+
+    async def task(flag_key):
+        # Creates a new isolation scope for the thread.
+        # This means the evaluations in each task are captured separately.
+        with sentry_sdk.isolation_scope():
+            add_feature_flag(flag_key, False)
+            # use a tag to identify to identify events later on
+            sentry_sdk.set_tag("task_id", flag_key)
+            sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    async def runner():
+        return asyncio.gather(task("world"), task("other"))
+
+    asyncio.run(runner())
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": False},
+            {"flag": "world", "result": False},
+        ]
+    }

tests/integrations/featureflags/test_featureflags.py

@@ -1,16 +1,15 @@
-import asyncio
 import concurrent.futures as cf
+import sys
 
 import ldclient
-
-import sentry_sdk
 import pytest
 
 from ldclient import LDClient
 from ldclient.config import Config
 from ldclient.context import Context
 from ldclient.integrations.test_data import TestData
 
+import sentry_sdk
 from sentry_sdk.integrations import DidNotEnable
 from sentry_sdk.integrations.launchdarkly import LaunchDarklyIntegration
 
@@ -19,9 +18,13 @@
     "use_global_client",
     (False, True),
 )
-def test_launchdarkly_integration(sentry_init, use_global_client):
+def test_launchdarkly_integration(
+    sentry_init, use_global_client, capture_events, uninstall_integration
+):
     td = TestData.data_source()
     config = Config("sdk-key", update_processor_class=td)
+
+    uninstall_integration(LaunchDarklyIntegration.identifier)
     if use_global_client:
         ldclient.set_config(config)
         sentry_init(integrations=[LaunchDarklyIntegration()])
@@ -39,60 +42,130 @@ def test_launchdarkly_integration(sentry_init, use_global_client):
     client.variation("world", Context.create("user1", "user"), False)
     client.variation("other", Context.create("user2", "user"), False)
 
-    assert sentry_sdk.get_current_scope().flags.get() == [
-        {"flag": "hello", "result": True},
-        {"flag": "world", "result": True},
-        {"flag": "other", "result": False},
-    ]
+    events = capture_events()
+    sentry_sdk.capture_exception(Exception("something wrong!"))
 
+    assert len(events) == 1
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "world", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
 
-def test_launchdarkly_integration_threaded(sentry_init):
+
+def test_launchdarkly_integration_threaded(
+    sentry_init, capture_events, uninstall_integration
+):
     td = TestData.data_source()
     client = LDClient(config=Config("sdk-key", update_processor_class=td))
-    sentry_init(integrations=[LaunchDarklyIntegration(ld_client=client)])
     context = Context.create("user1")
 
+    uninstall_integration(LaunchDarklyIntegration.identifier)
+    sentry_init(integrations=[LaunchDarklyIntegration(ld_client=client)])
+    events = capture_events()
+
     def task(flag_key):
         # Creates a new isolation scope for the thread.
         # This means the evaluations in each task are captured separately.
         with sentry_sdk.isolation_scope():
             client.variation(flag_key, context, False)
-            return [f["flag"] for f in sentry_sdk.get_current_scope().flags.get()]
+            # use a tag to identify to identify events later on
+            sentry_sdk.set_tag("task_id", flag_key)
+            sentry_sdk.capture_exception(Exception("something wrong!"))
 
     td.update(td.flag("hello").variation_for_all(True))
     td.update(td.flag("world").variation_for_all(False))
     # Capture an eval before we split isolation scopes.
     client.variation("hello", context, False)
 
     with cf.ThreadPoolExecutor(max_workers=2) as pool:
-        results = list(pool.map(task, ["world", "other"]))
-
-    assert results[0] == ["hello", "world"]
-    assert results[1] == ["hello", "other"]
+        pool.map(task, ["world", "other"])
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "world", "result": False},
+        ]
+    }
+
+
+@pytest.mark.skipif(sys.version_info < (3, 7), reason="requires python3.7 or higher")
+def test_launchdarkly_integration_asyncio(
+    sentry_init, capture_events, uninstall_integration
+):
+    """Assert concurrently evaluated flags do not pollute one another."""
 
+    asyncio = pytest.importorskip("asyncio")
 
-def test_launchdarkly_integration_asyncio(sentry_init):
-    """Assert concurrently evaluated flags do not pollute one another."""
     td = TestData.data_source()
     client = LDClient(config=Config("sdk-key", update_processor_class=td))
-    sentry_init(integrations=[LaunchDarklyIntegration(ld_client=client)])
     context = Context.create("user1")
 
+    uninstall_integration(LaunchDarklyIntegration.identifier)
+    sentry_init(integrations=[LaunchDarklyIntegration(ld_client=client)])
+    events = capture_events()
+
     async def task(flag_key):
         with sentry_sdk.isolation_scope():
             client.variation(flag_key, context, False)
-            return [f["flag"] for f in sentry_sdk.get_current_scope().flags.get()]
+            # use a tag to identify to identify events later on
+            sentry_sdk.set_tag("task_id", flag_key)
+            sentry_sdk.capture_exception(Exception("something wrong!"))
 
     async def runner():
         return asyncio.gather(task("world"), task("other"))
 
     td.update(td.flag("hello").variation_for_all(True))
     td.update(td.flag("world").variation_for_all(False))
+    # Capture an eval before we split isolation scopes.
     client.variation("hello", context, False)
 
-    results = asyncio.run(runner()).result()
-    assert results[0] == ["hello", "world"]
-    assert results[1] == ["hello", "other"]
+    asyncio.run(runner())
+
+    # Capture error in original scope
+    sentry_sdk.set_tag("task_id", "0")
+    sentry_sdk.capture_exception(Exception("something wrong!"))
+
+    assert len(events) == 3
+    events.sort(key=lambda e: e["tags"]["task_id"])
+
+    assert events[0]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+        ]
+    }
+    assert events[1]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "other", "result": False},
+        ]
+    }
+    assert events[2]["contexts"]["flags"] == {
+        "values": [
+            {"flag": "hello", "result": True},
+            {"flag": "world", "result": False},
+        ]
+    }
 
 
 def test_launchdarkly_integration_did_not_enable(monkeypatch):