Add w3c traceparent header support (#1448)

* Add config settings

* Add traceparent logic

* Log warning if value is not recognized

* Move & rename function_not_called

* Use existing function called validators

* Fixup: lint log message

* Fixup: validator import paths

* Fixup: validator import path

* Fixup: validator import path

* Reformat w/ ruff

* Use legacy bitnami for now (#1471)

* Use legacy bitnami for now

* Revert solr change

* Revert zookeeper change

* Add graphene-django instrumentation (#1451)

* Add graphene-django instrumentation

* Increase naming priority

* Remove unused import

* Add sychronous schema tests

* Clean up test files

* Remove commented out code

* Megalinter fixes

* Add operation & resolver tests

* Refine tests

* MegaLinter fixes

* Suggested reviewer changes

* Megalinter fixes

* Fixup: paths

* Fixup: reformat lint changes

---------

Co-authored-by: mergify[bot] <37929162+mergify[bot]@users.noreply.github.com>
Co-authored-by: Lalleh Rafeei <[email protected]>

Author: 3 people

newrelic/api/transaction.py

@@ -286,6 +286,7 @@ def __init__(self, application, enabled=None, source=None):
         self.tracestate = ""
         self._priority = None
         self._sampled = None
+        self._traceparent_sampled = None
 
         self._distributed_trace_state = 0
 
@@ -1004,16 +1005,36 @@ def _update_agent_attributes(self):
     def user_attributes(self):
         return create_attributes(self._custom_params, DST_ALL, self.attribute_filter)
 
-    def _compute_sampled_and_priority(self):
+    def sampling_algo_compute_sampled_and_priority(self):
         if self._priority is None:
-            # truncate priority field to 6 digits past the decimal
+            # Truncate priority field to 6 digits past the decimal.
             self._priority = float(f"{random.random():.6f}")  # noqa: S311
-
         if self._sampled is None:
             self._sampled = self._application.compute_sampled()
             if self._sampled:
                 self._priority += 1
 
+    def _compute_sampled_and_priority(self):
+        if self._traceparent_sampled is None:
+            config = "default"  # Use sampling algo.
+        elif self._traceparent_sampled:
+            setting_path = "distributed_tracing.sampler.remote_parent_sampled"
+            config = self.settings.distributed_tracing.sampler.remote_parent_sampled
+        else:  # self._traceparent_sampled is False.
+            setting_path = "distributed_tracing.sampler.remote_parent_not_sampled"
+            config = self.settings.distributed_tracing.sampler.remote_parent_not_sampled
+
+        if config == "always_on":
+            self._sampled = True
+            self._priority = 2.0
+        elif config == "always_off":
+            self._sampled = False
+            self._priority = 0
+        else:
+            if config != "default":
+                _logger.warning("%s=%s is not a recognized value. Using 'default' instead.", setting_path, config)
+            self.sampling_algo_compute_sampled_and_priority()
+
     def _freeze_path(self):
         if self._frozen_path is None:
             self._name_priority = None
@@ -1348,6 +1369,7 @@ def accept_distributed_trace_headers(self, headers, transport_type="HTTP"):
                     else:
                         self._record_supportability("Supportability/TraceContext/TraceState/NoNrEntry")
 
+            self._traceparent_sampled = data.get("sa")
             self._accept_distributed_trace_data(data, transport_type)
             self._record_supportability("Supportability/TraceContext/Accept/Success")
             return True

newrelic/common/encoding_utils.py

@@ -34,6 +34,7 @@
 DELIMITER_FORMAT_RE = re.compile("[ \t]*,[ \t]*")
 PARENT_TYPE = {"0": "App", "1": "Browser", "2": "Mobile"}
 BASE64_DECODE_STR = getattr(base64, "decodestring", None)
+FLAG_SAMPLED = 1
 
 
 # Functions for encoding/decoding JSON. These wrappers are used in order
@@ -455,7 +456,10 @@ def decode(cls, payload):
         if parent_id == "0" * 16 or trace_id == "0" * 32:
             return None
 
-        return cls(tr=trace_id, id=parent_id)
+        # Sampled flag
+        sa = bool(int(fields[3], 2) & FLAG_SAMPLED)
+
+        return cls(tr=trace_id, id=parent_id, sa=sa)
 
 
 class W3CTraceState(OrderedDict):

newrelic/config.py

@@ -404,6 +404,8 @@ def _process_configuration(section):
     _process_setting(section, "ml_insights_events.enabled", "getboolean", None)
     _process_setting(section, "distributed_tracing.enabled", "getboolean", None)
     _process_setting(section, "distributed_tracing.exclude_newrelic_header", "getboolean", None)
+    _process_setting(section, "distributed_tracing.sampler.remote_parent_sampled", "get", None)
+    _process_setting(section, "distributed_tracing.sampler.remote_parent_not_sampled", "get", None)
     _process_setting(section, "span_events.enabled", "getboolean", None)
     _process_setting(section, "span_events.max_samples_stored", "getint", None)
     _process_setting(section, "span_events.attributes.enabled", "getboolean", None)

newrelic/core/config.py

@@ -332,6 +332,10 @@ class DistributedTracingSettings(Settings):
     pass
 
 
+class DistributedTracingSamplerSettings(Settings):
+    pass
+
+
 class ServerlessModeSettings(Settings):
     pass
 
@@ -501,6 +505,7 @@ class EventHarvestConfigHarvestLimitSettings(Settings):
 _settings.datastore_tracer.instance_reporting = DatastoreTracerInstanceReportingSettings()
 _settings.debug = DebugSettings()
 _settings.distributed_tracing = DistributedTracingSettings()
+_settings.distributed_tracing.sampler = DistributedTracingSamplerSettings()
 _settings.error_collector = ErrorCollectorSettings()
 _settings.error_collector.attributes = ErrorCollectorAttributesSettings()
 _settings.event_harvest_config = EventHarvestConfigSettings()
@@ -828,6 +833,12 @@ def default_otlp_host(host):
 _settings.ml_insights_events.enabled = False
 
 _settings.distributed_tracing.enabled = _environ_as_bool("NEW_RELIC_DISTRIBUTED_TRACING_ENABLED", default=True)
+_settings.distributed_tracing.sampler.remote_parent_sampled = os.environ.get(
+    "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_SAMPLED", "default"
+)
+_settings.distributed_tracing.sampler.remote_parent_not_sampled = os.environ.get(
+    "NEW_RELIC_DISTRIBUTED_TRACING_SAMPLER_REMOTE_PARENT_NOT_SAMPLED", "default"
+)
 _settings.distributed_tracing.exclude_newrelic_header = False
 _settings.span_events.enabled = _environ_as_bool("NEW_RELIC_SPAN_EVENTS_ENABLED", default=True)
 _settings.span_events.attributes.enabled = True

tests/agent_features/test_async_context_propagation.py

@@ -13,7 +13,8 @@
 # limitations under the License.
 
 import pytest
-from testing_support.fixtures import function_not_called, override_generic_settings
+from testing_support.fixtures import override_generic_settings
+from testing_support.validators.validate_function_not_called import validate_function_not_called
 from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics
 
 from newrelic.api.application import application_instance as application
@@ -128,7 +129,7 @@ def handle_exception(loop, context):
 
 
 @override_generic_settings(global_settings(), {"enabled": False})
-@function_not_called("newrelic.core.stats_engine", "StatsEngine.record_transaction")
+@validate_function_not_called("newrelic.core.stats_engine", "StatsEngine.record_transaction")
 def test_nr_disabled(event_loop):
     import asyncio
 

tests/agent_features/test_custom_events.py

@@ -15,12 +15,13 @@
 import time
 
 import pytest
-from testing_support.fixtures import function_not_called, override_application_settings, reset_core_stats_engine
+from testing_support.fixtures import override_application_settings, reset_core_stats_engine
 from testing_support.validators.validate_custom_event import (
     validate_custom_event_count,
     validate_custom_event_in_application_stats_engine,
 )
 from testing_support.validators.validate_custom_events import validate_custom_events
+from testing_support.validators.validate_function_not_called import validate_function_not_called
 
 from newrelic.api.application import application_instance as application
 from newrelic.api.background_task import background_task
@@ -214,14 +215,14 @@ def test_custom_event_settings_check_custom_insights_enabled():
 
 
 @override_application_settings({"custom_insights_events.enabled": False})
-@function_not_called("newrelic.api.transaction", "create_custom_event")
+@validate_function_not_called("newrelic.api.transaction", "create_custom_event")
 @background_task()
 def test_transaction_create_custom_event_not_called():
     record_custom_event("FooEvent", _user_params)
 
 
 @override_application_settings({"custom_insights_events.enabled": False})
-@function_not_called("newrelic.core.application", "create_custom_event")
+@validate_function_not_called("newrelic.core.application", "create_custom_event")
 @background_task()
 def test_application_create_custom_event_not_called():
     app = application()

tests/agent_features/test_distributed_tracing.py

@@ -17,8 +17,10 @@
 
 import pytest
 import webtest
-from testing_support.fixtures import override_application_settings, validate_attributes
+from testing_support.fixtures import override_application_settings, validate_attributes, validate_attributes_complete
 from testing_support.validators.validate_error_event_attributes import validate_error_event_attributes
+from testing_support.validators.validate_function_called import validate_function_called
+from testing_support.validators.validate_function_not_called import validate_function_not_called
 from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes
 from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics
 
@@ -36,6 +38,7 @@
 )
 from newrelic.api.web_transaction import WSGIWebTransaction
 from newrelic.api.wsgi_application import wsgi_application
+from newrelic.core.attribute import Attribute
 
 distributed_trace_intrinsics = ["guid", "traceId", "priority", "sampled"]
 inbound_payload_intrinsics = [
@@ -410,3 +413,65 @@ def _test_inbound_dt_payload_acceptance():
             assert not result
 
     _test_inbound_dt_payload_acceptance()
+
+
+@pytest.mark.parametrize(
+    "sampled,remote_parent_sampled,remote_parent_not_sampled,expected_sampled,expected_priority,expected_adaptive_sampling_algo_called",
+    (
+        (True, "default", "default", None, None, True),  # Uses sampling algo.
+        (True, "always_on", "default", True, 2, False),  # Always sampled.
+        (True, "always_off", "default", False, 0, False),  # Never sampled.
+        (False, "default", "default", None, None, True),  # Uses sampling algo.
+        (False, "always_on", "default", None, None, True),  # Uses sampling alog.
+        (False, "always_off", "default", None, None, True),  # Uses sampling algo.
+        (True, "default", "always_on", None, None, True),  # Uses sampling algo.
+        (True, "default", "always_off", None, None, True),  # Uses sampling algo.
+        (False, "default", "always_on", True, 2, False),  # Always sampled.
+        (False, "default", "always_off", False, 0, False),  # Never sampled.
+    ),
+)
+def test_distributed_trace_w3cparent_sampling_decision(
+    sampled,
+    remote_parent_sampled,
+    remote_parent_not_sampled,
+    expected_sampled,
+    expected_priority,
+    expected_adaptive_sampling_algo_called,
+):
+    required_intrinsics = []
+    if expected_sampled is not None:
+        required_intrinsics.append(Attribute(name="sampled", value=expected_sampled, destinations=0b110))
+    if expected_priority is not None:
+        required_intrinsics.append(Attribute(name="priority", value=expected_priority, destinations=0b110))
+
+    test_settings = _override_settings.copy()
+    test_settings.update(
+        {
+            "distributed_tracing.sampler.remote_parent_sampled": remote_parent_sampled,
+            "distributed_tracing.sampler.remote_parent_not_sampled": remote_parent_not_sampled,
+            "span_events.enabled": True,
+        }
+    )
+    if expected_adaptive_sampling_algo_called:
+        function_called_decorator = validate_function_called(
+            "newrelic.api.transaction", "Transaction.sampling_algo_compute_sampled_and_priority"
+        )
+    else:
+        function_called_decorator = validate_function_not_called(
+            "newrelic.api.transaction", "Transaction.sampling_algo_compute_sampled_and_priority"
+        )
+
+    @function_called_decorator
+    @override_application_settings(test_settings)
+    @validate_attributes_complete("intrinsic", required_intrinsics)
+    @background_task(name="test_distributed_trace_attributes")
+    def _test():
+        txn = current_transaction()
+
+        headers = {
+            "traceparent": f"00-0af7651916cd43dd8448eb211c80319c-00f067aa0ba902b7-{int(sampled):02x}",
+            "tracestate": "rojo=f06a0ba902b7,congo=t61rcWkgMzE",
+        }
+        accept_distributed_trace_headers(headers)
+
+    _test()

tests/agent_features/test_ml_events.py

@@ -16,7 +16,8 @@
 from importlib import reload
 
 import pytest
-from testing_support.fixtures import function_not_called, override_application_settings, reset_core_stats_engine
+from testing_support.fixtures import override_application_settings, reset_core_stats_engine
+from testing_support.validators.validate_function_not_called import validate_function_not_called
 from testing_support.validators.validate_ml_event_count import validate_ml_event_count
 from testing_support.validators.validate_ml_event_payload import validate_ml_event_payload
 from testing_support.validators.validate_ml_events import validate_ml_events
@@ -331,15 +332,15 @@ def test_ml_event_settings_check_ml_insights_disabled():
 
 @override_application_settings({"ml_insights_events.enabled": False})
 @reset_core_stats_engine()
-@function_not_called("newrelic.api.transaction", "create_custom_event")
+@validate_function_not_called("newrelic.api.transaction", "create_custom_event")
 @background_task()
 def test_transaction_create_ml_event_not_called():
     record_ml_event("FooEvent", {"foo": "bar"})
 
 
 @override_application_settings({"ml_insights_events.enabled": False})
 @reset_core_stats_engine()
-@function_not_called("newrelic.core.application", "create_custom_event")
+@validate_function_not_called("newrelic.core.application", "create_custom_event")
 @background_task()
 def test_application_create_ml_event_not_called():
     app = application()

tests/agent_features/test_span_events.py

@@ -15,7 +15,8 @@
 import sys
 
 import pytest
-from testing_support.fixtures import dt_enabled, function_not_called, override_application_settings
+from testing_support.fixtures import dt_enabled, override_application_settings
+from testing_support.validators.validate_function_not_called import validate_function_not_called
 from testing_support.validators.validate_span_events import validate_span_events
 from testing_support.validators.validate_transaction_event_attributes import validate_transaction_event_attributes
 from testing_support.validators.validate_transaction_metrics import validate_transaction_metrics
@@ -415,7 +416,7 @@ def _test():
             pass
 
     if not spans_expected:
-        _test = function_not_called("newrelic.core.attribute", "resolve_agent_attributes")(_test)
+        _test = validate_function_not_called("newrelic.core.attribute", "resolve_agent_attributes")(_test)
 
     _test()
 

tests/agent_unittests/test_harvest_loop.py

@@ -18,7 +18,8 @@
 from pathlib import Path
 
 import pytest
-from testing_support.fixtures import failing_endpoint, function_not_called, override_generic_settings
+from testing_support.fixtures import failing_endpoint, override_generic_settings
+from testing_support.validators.validate_function_not_called import validate_function_not_called
 
 from newrelic.common.agent_http import DeveloperModeClient
 from newrelic.common.object_wrapper import function_wrapper, transient_function_wrapper
@@ -646,7 +647,7 @@ def test_serverless_mode_adaptive_sampling(time_to_next_reset, computed_count, c
     assert app.adaptive_sampler.computed_count_last == computed_count_last
 
 
-@function_not_called("newrelic.core.adaptive_sampler", "AdaptiveSampler._reset")
+@validate_function_not_called("newrelic.core.adaptive_sampler", "AdaptiveSampler._reset")
 @override_generic_settings(settings, {"developer_mode": True})
 def test_compute_sampled_no_reset():
     app = Application("Python Agent Test (Harvest Loop)")