feat(agent): Add sampling control based on upstream sampling decisions (#1104)

This implements the sampling spec to allow control over the agent
sampling decisions based on the value in the sampled flag of an inbound
traceparent header.

---------

Co-authored-by: Michal Nowacki <[email protected]>

Author: ZNeumann

agent/php_newrelic.h

@@ -494,6 +494,16 @@ nrinibool_t span_events_enabled; /* newrelic.span_events_enabled */
 nriniuint_t
     span_events_max_samples_stored; /* newrelic.span_events.max_samples_stored
                                      */
+
+nrinistr_t dt_remote_parent_sampled; /* newrelic.distributed_tracing.sampler.remote_parent_sampled */
+nrinistr_t
+    dt_remote_parent_not_sampled; /* newrelic.distributed_tracing.sampler.remote_parent_not_sampled */
+/* decoding of newrelic.distributed_tracing.sampler.remote_parent_sampled and
+ * newrelic.distributed_tracing.sampler.remote_parent_not_sampled.
+ */
+nr_upstream_parent_sampling_control_t dt_sampler_parent_sampled;
+nr_upstream_parent_sampling_control_t dt_sampler_parent_not_sampled;
+
 nrinistr_t
     trace_observer_host; /* newrelic.infinite_tracing.trace_observer.host */
 nriniuint_t

agent/php_nrini.c

@@ -1970,6 +1970,55 @@ static PHP_INI_MH(nr_wordpress_hooks_options_mh) {
   return SUCCESS;
 }
 
+static PHP_INI_MH(nr_dt_sampler_remote_parent_mh) {
+  nrinistr_t* p;
+
+  char* base = (char*)mh_arg2;
+  p = (nrinistr_t*)(base + (size_t)mh_arg1);
+  bool parent_sampled = false;
+
+  (void)mh_arg3;
+  NR_UNUSED_TSRMLS;
+
+  p->where = 0;
+
+  if (0 == nr_strcmp(ZEND_STRING_VALUE(entry->name),
+                     "newrelic.distributed_tracing.sampler.remote_parent_sampled")) {
+    parent_sampled = true;
+  }
+
+  if (0 == nr_strcmp(NEW_VALUE, "default")) {
+    if (parent_sampled) {
+      NRPRG(dt_sampler_parent_sampled) = DEFAULT;
+    } else {
+      NRPRG(dt_sampler_parent_not_sampled) = DEFAULT;
+    }
+  } else if (0 == nr_strcmp(NEW_VALUE, "always_on")) {
+    if (parent_sampled) {
+      NRPRG(dt_sampler_parent_sampled) = ALWAYS_KEEP;
+    } else {
+      NRPRG(dt_sampler_parent_not_sampled) = ALWAYS_KEEP;
+    }
+  } else if (0 == nr_strcmp(NEW_VALUE, "always_off")) {
+    if (parent_sampled) {
+      NRPRG(dt_sampler_parent_sampled) = ALWAYS_DROP;
+    } else {
+      NRPRG(dt_sampler_parent_not_sampled) = ALWAYS_DROP;
+    }
+  } else {
+    nrl_warning(NRL_INIT, "Invalid %s value \"%s\"; using \"%s\" instead.",
+                ZEND_STRING_VALUE(entry->name), NEW_VALUE,
+                DEFAULT_WORDPRESS_HOOKS_OPTIONS);
+    /* This will cause PHP to call the handler again with default value */
+    return FAILURE;
+  }
+
+  p->where = stage;
+  p->value = NEW_VALUE;
+
+  return SUCCESS;
+}
+
 /*
  * Now for the actual INI entry table. Please note there are two types of INI
  * entry specification used.
@@ -2938,6 +2987,24 @@ STD_PHP_INI_ENTRY_EX("newrelic.distributed_tracing_exclude_newrelic_header",
                      newrelic_globals,
                      0)
 
+
+STD_PHP_INI_ENTRY_EX("newrelic.distributed_tracing.sampler.remote_parent_sampled",
+                     "default",
+                     NR_PHP_REQUEST,
+                     nr_dt_sampler_remote_parent_mh,
+                     dt_remote_parent_sampled,
+                     zend_newrelic_globals,
+                     newrelic_globals,
+                     0)
+STD_PHP_INI_ENTRY_EX("newrelic.distributed_tracing.sampler.remote_parent_not_sampled",
+                     "default",
+                     NR_PHP_REQUEST,
+                     nr_dt_sampler_remote_parent_mh,
+                     dt_remote_parent_not_sampled,
+                     zend_newrelic_globals,
+                     newrelic_globals,
+                     0)
+
 /*
  * This setting is not documented and affects the length of the interally used
  * trace id. This INI setting should not be modified unless requested by

agent/php_txn.c

@@ -942,6 +942,10 @@ nr_status_t nr_php_txn_begin(const char* appnames,
       = is_cli ? NRINI(tt_max_segments_cli) : NRINI(tt_max_segments_web);
   opts.span_queue_batch_size = NRINI(agent_span_queue_size);
   opts.span_queue_batch_timeout = NRINI(agent_span_queue_timeout);
+  opts.dt_sampler_parent_sampled
+      = NRPRG(dt_sampler_parent_sampled);
+  opts.dt_sampler_parent_not_sampled
+      = NRPRG(dt_sampler_parent_not_sampled);
   opts.logging_enabled = NRINI(logging_enabled);
   opts.log_decorating_enabled = NRINI(log_decorating_enabled);
   opts.log_forwarding_enabled = NRINI(log_forwarding_enabled);

agent/scripts/newrelic.ini.template

@@ -927,6 +927,36 @@ newrelic.daemon.logfile = "/var/log/newrelic/newrelic-daemon.log"
 ;
 ;newrelic.distributed_tracing_exclude_newrelic_header = false
 
+; Setting: newrelic.distributed_tracing.sampler.remote_parent_sampled
+; Type   : string
+; Scope  : per-directory
+; Default: "default"
+; Info   : This option defines how the agent should handle sampling spans when
+;          their parent span from an upstream entity was sampled. For example,
+;          setting remote_parent_sampled: always_on means the agent will sample
+;          anything if the upstream entity sampled the parent.
+;          The possible values are:
+;          - "default": Use New Relic's standard sampling rules.
+;          - "always_on": Always sample spans whose upstream parent was not sampled.
+;          - "always_off": Always skip sampling spans whose upstream parent was not sampled.
+;
+;newrelic.distributed_tracing.sampler.remote_parent_sampled = "default"
+
+; Setting: newrelic.distributed_tracing.sampler.remote_parent_not_sampled
+; Type   : string
+; Scope  : per-directory
+; Default: "default"
+; Info   : This option defines how the agent should handle sampling spans when
+;          their parent span from an upstream entity was not sampled. For example,
+;          setting remote_parent_not_sampled: always_off means the agent will not
+;          try to sample anything if the upstream entity did not sample the parent.
+;          The possible values are:
+;          - "default": Use New Relic's standard sampling rules.
+;          - "always_on": Always sample spans whose upstream parent was not sampled.
+;          - "always_off": Always skip sampling spans whose upstream parent was not sampled.
+;
+;newrelic.distributed_tracing.sampler.remote_parent_sampled = "default"
+
 ; Setting: newrelic.span_events_enabled
 ; Type   : boolean
 ; Scope  : per-directory

axiom/nr_distributed_trace.c

@@ -1298,3 +1298,43 @@ char* nr_distributed_trace_create_w3c_traceparent_header(const char* trace_id,
 
   return trace_parent_header;
 }
+
+void nr_distributed_trace_handle_inbound_w3c_sampled_flag(
+        nr_distributed_trace_t* dt,
+        const nrobj_t* trace_headers,
+    nr_upstream_parent_sampling_control_t remote_parent_sampled,
+    nr_upstream_parent_sampling_control_t remote_parent_not_sampled) {
+  const nrobj_t* traceparent = NULL;
+  int sampled = 0;
+  nr_status_t parse_err = NR_FAILURE;
+  if (DEFAULT != remote_parent_sampled || DEFAULT != remote_parent_not_sampled) {
+    traceparent = nro_get_hash_value(trace_headers, "traceparent", &parse_err);
+    if (nrunlikely(NULL == traceparent || NR_SUCCESS != parse_err)) {
+      return;
+    }
+    sampled = nro_get_hash_int(traceparent, "trace_flags", &parse_err);
+    if (nrunlikely(NR_SUCCESS != parse_err)) {
+      return;
+    }
+    /* The final bit of the trace_flags indicates the sampling decision */
+    if (sampled & 0x01) {
+      if (DEFAULT != remote_parent_sampled) {
+        if (ALWAYS_KEEP == remote_parent_sampled) {
+          dt->sampled = true;
+          dt->priority = 2;
+        } else {
+          dt->sampled = false;
+        }
+      }
+    } else {
+      if (DEFAULT != remote_parent_not_sampled) {
+        if (ALWAYS_DROP == remote_parent_not_sampled) {
+          dt->sampled = false;
+        } else {
+          dt->sampled = true;
+          dt->priority = 2;
+        }
+      }
+    }
+  }
+}

axiom/nr_distributed_trace.h

@@ -60,6 +60,16 @@ static const char NR_DISTRIBUTED_TRACE_W3C_TRACECONTEXT_ACCEPT_EXCEPTION[]
 typedef struct _nr_distributed_trace_t nr_distributed_trace_t;
 typedef struct _nr_distributed_trace_payload_t nr_distributed_trace_payload_t;
 
+/*
+ * Control options for how to respect other-vendor upstream sampling
+ * decisions.
+ */
+typedef enum {
+  DEFAULT,
+  ALWAYS_KEEP,
+  ALWAYS_DROP
+} nr_upstream_parent_sampling_control_t;
+
 /*
  * Purpose : Creates/allocates a new distributed tracing metadata struct
  *           instance.  It's the responsibility of the caller to
@@ -409,4 +419,18 @@ bool nr_distributed_trace_accept_inbound_w3c_payload(
     const char* transport_type,
     const char** error);
 
+/*
+ * Purpose : Handle upstream w3c sampled flag according to settings
+ *
+ * Params : 1. The distributed trace object
+ *          2. W3C trace headers objet
+ *          3. Setting for if the upstream trace is sampled
+ *          4. Setting for if the upstream trace is not sampled
+ */
+void nr_distributed_trace_handle_inbound_w3c_sampled_flag(
+    nr_distributed_trace_t* dt,
+    const nrobj_t* trace_headers,
+    nr_upstream_parent_sampling_control_t remote_parent_sampled,
+    nr_upstream_parent_sampling_control_t remote_parent_not_sampled);
+
 #endif /* NR_DISTRIBUTED_TRACE_HDR */

axiom/nr_txn.c

@@ -2972,6 +2972,14 @@ static bool nr_txn_accept_w3c_trace_context_headers(
   nr_distributed_trace_accept_inbound_w3c_payload(
       txn->distributed_trace, trace_headers, transport_type, &error_metrics);
 
+  /* Depending on the user's INI settings, we may or may not want to
+   * consider the traceparent's sampled field */
+  nr_distributed_trace_handle_inbound_w3c_sampled_flag(
+      txn->distributed_trace,
+      trace_headers,
+      txn->options.dt_sampler_parent_sampled,
+      txn->options.dt_sampler_parent_not_sampled);
+
   if (error_metrics) {
     nr_txn_force_single_count(txn, error_metrics);
   }

axiom/nr_txn.h

@@ -100,6 +100,14 @@ typedef struct _nrtxnopt_t {
                                                        headers in favor of only
                                                        W3C trace context headers
                                                      */
+  nr_upstream_parent_sampling_control_t
+    dt_sampler_parent_sampled; /* how to sample spans when non-
+                                  New Relic upstream did sample.
+                                */
+  nr_upstream_parent_sampling_control_t
+    dt_sampler_parent_not_sampled; /* how to sample spans when non-
+                                      New Relic upstream didn't sample.
+                                    */
   int span_events_enabled; /* Whether span events are enabled */
   size_t
       span_events_max_samples_stored; /* The maximum number of span events per