Skip to content

Commit 47c2d39

Browse files
trasktrask
authored
Fix sampling rate recorded for dependencies (#1582)
1 parent 3c7da58 commit 47c2d39

File tree

10 files changed

+346
-69
lines changed

10 files changed

+346
-69
lines changed

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/instrumentation/sdk/BytecodeUtilImpl.java‎

Lines changed: 23 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import java.util.concurrent.atomic.AtomicBoolean;
2828

2929
import com.google.common.base.Strings;
30+
import com.microsoft.applicationinsights.agent.Exporter;
3031
import com.microsoft.applicationinsights.agent.bootstrap.BytecodeUtil.BytecodeUtilDelegate;
3132
import com.microsoft.applicationinsights.agent.internal.Global;
3233
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingScoreGeneratorV2;
@@ -220,20 +221,31 @@ public void logErrorOnce(Throwable t) {
220221

221222
private static void track(Telemetry telemetry) {
222223
SpanContext context = Span.current().getSpanContext();
224+
double samplingPercentage;
223225
if (context.isValid()) {
224-
String traceId = context.getTraceId();
225-
String spanId = context.getSpanId();
226-
telemetry.getContext().getOperation().setId(traceId);
227-
telemetry.getContext().getOperation().setParentId(spanId);
228-
}
229-
double samplingPercentage = Global.getSamplingPercentage();
230-
if (sample(telemetry, samplingPercentage)) {
231-
if (telemetry instanceof SupportSampling && samplingPercentage != 100) {
232-
((SupportSampling) telemetry).setSamplingPercentage(samplingPercentage);
226+
if (!context.isSampled()) {
227+
// sampled out
228+
return;
233229
}
234-
// this is not null because sdk instrumentation is not added until Global.setTelemetryClient() is called
235-
checkNotNull(Global.getTelemetryClient()).track(telemetry);
230+
telemetry.getContext().getOperation().setId(context.getTraceId());
231+
telemetry.getContext().getOperation().setParentId(context.getSpanId());
232+
samplingPercentage =
233+
Exporter.getSamplingPercentage(context.getTraceState(), Global.getSamplingPercentage(), false);
234+
} else {
235+
// sampling is done using the configured sampling percentage
236+
samplingPercentage = Global.getSamplingPercentage();
237+
if (!sample(telemetry, samplingPercentage)) {
238+
// sampled out
239+
return;
240+
}
241+
}
242+
// sampled in
243+
244+
if (telemetry instanceof SupportSampling && samplingPercentage != 100) {
245+
((SupportSampling) telemetry).setSamplingPercentage(samplingPercentage);
236246
}
247+
// this is not null because sdk instrumentation is not added until Global.setTelemetryClient() is called
248+
checkNotNull(Global.getTelemetryClient()).track(telemetry);
237249
}
238250

239251
private static boolean sample(Telemetry telemetry, double samplingPercentage) {

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/propagator/DelegatingPropagator.java‎

Lines changed: 81 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,11 @@
33
import java.util.Collection;
44
import javax.annotation.Nullable;
55

6+
import com.microsoft.applicationinsights.agent.Exporter;
7+
import io.opentelemetry.api.trace.Span;
8+
import io.opentelemetry.api.trace.SpanContext;
9+
import io.opentelemetry.api.trace.TraceFlags;
10+
import io.opentelemetry.api.trace.TraceState;
611
import io.opentelemetry.api.trace.propagation.W3CTraceContextPropagator;
712
import io.opentelemetry.context.Context;
813
import io.opentelemetry.context.propagation.TextMapGetter;
@@ -21,8 +26,13 @@ public static DelegatingPropagator getInstance() {
2126
}
2227

2328
public void setUpStandardDelegate() {
29+
30+
// TODO when should we enable baggage propagation?
31+
// currently using modified W3CTraceContextPropagator because "ai-internal-sp" trace state
32+
// shouldn't be sent over the wire (at least not yet, and not with that name)
33+
2434
// important that W3CTraceContextPropagator is last, so it will take precedence if both sets of headers are present
25-
delegate = TextMapPropagator.composite(AiLegacyPropagator.getInstance(), W3CTraceContextPropagator.getInstance());
35+
delegate = TextMapPropagator.composite(AiLegacyPropagator.getInstance(), new ModifiedW3CTraceContextPropagator());
2636
}
2737

2838
@Override
@@ -39,4 +49,74 @@ public <C> void inject(Context context, @Nullable C carrier, TextMapSetter<C> se
3949
public <C> Context extract(Context context, @Nullable C carrier, TextMapGetter<C> getter) {
4050
return delegate.extract(context, carrier, getter);
4151
}
52+
53+
private static class ModifiedW3CTraceContextPropagator implements TextMapPropagator {
54+
55+
private final TextMapPropagator delegate = W3CTraceContextPropagator.getInstance();
56+
57+
@Override
58+
public Collection<String> fields() {
59+
return delegate.fields();
60+
}
61+
62+
@Override
63+
public <C> void inject(Context context, @Nullable C carrier, TextMapSetter<C> setter) {
64+
// do not propagate sampling percentage downstream YET
65+
SpanContext spanContext = Span.fromContext(context).getSpanContext();
66+
// sampling percentage should always be present, so no need to optimize with checking if present
67+
TraceState traceState = spanContext.getTraceState();
68+
TraceState updatedTraceState;
69+
if (traceState.size() == 1 && traceState.get(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE) != null) {
70+
// this is a common case, worth optimizing
71+
updatedTraceState = TraceState.getDefault();
72+
} else {
73+
updatedTraceState = traceState.toBuilder()
74+
.remove(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE)
75+
.build();
76+
}
77+
SpanContext updatedSpanContext = new ModifiedSpanContext(spanContext, updatedTraceState);
78+
delegate.inject(Context.root().with(Span.wrap(updatedSpanContext)), carrier, setter);
79+
}
80+
81+
@Override
82+
public <C> Context extract(Context context, @Nullable C carrier, TextMapGetter<C> getter) {
83+
return delegate.extract(context, carrier, getter);
84+
}
85+
}
86+
87+
private static class ModifiedSpanContext implements SpanContext {
88+
89+
private final SpanContext delegate;
90+
private final TraceState traceState;
91+
92+
private ModifiedSpanContext(SpanContext delegate, TraceState traceState) {
93+
this.delegate = delegate;
94+
this.traceState = traceState;
95+
}
96+
97+
@Override
98+
public String getTraceId() {
99+
return delegate.getTraceId();
100+
}
101+
102+
@Override
103+
public String getSpanId() {
104+
return delegate.getSpanId();
105+
}
106+
107+
@Override
108+
public TraceFlags getTraceFlags() {
109+
return delegate.getTraceFlags();
110+
}
111+
112+
@Override
113+
public TraceState getTraceState() {
114+
return traceState;
115+
}
116+
117+
@Override
118+
public boolean isRemote() {
119+
return delegate.isRemote();
120+
}
121+
}
42122
}

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java‎

Lines changed: 15 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package com.microsoft.applicationinsights.agent.internal.sampling;
22

33
import java.util.List;
4-
import javax.annotation.Nullable;
54

65
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides.MatcherGroup;
76
import io.opentelemetry.api.common.Attributes;
@@ -27,7 +26,7 @@ class AiSampler implements Sampler {
2726
//
2827
// failure to follow this pattern can result in unexpected / incorrect computation of values in the portal
2928
private final double defaultSamplingPercentage;
30-
private final SamplingResult defaultRecordAndSampleResult;
29+
private final SamplingResult recordAndSampleAndAddTraceStateIfMissing;
3130

3231
private final SamplingOverrides samplingOverrides;
3332

@@ -38,12 +37,13 @@ class AiSampler implements Sampler {
3837
// samplingPercentage is still used in BehaviorIfNoMatchingOverrides.RECORD_AND_SAMPLE
3938
// to set an approximate value for the span attribute "applicationinsights.internal.sampling_percentage"
4039
//
41-
// in the future the sampling percentage (or its invert "count") will be
40+
// in the future the sampling percentage (or its inverse "count") will be
4241
// carried down by trace state to set the accurate value
4342
AiSampler(double samplingPercentage, SamplingOverrides samplingOverrides,
4443
BehaviorIfNoMatchingOverrides behaviorIfNoMatchingOverrides) {
4544
this.defaultSamplingPercentage = samplingPercentage;
46-
defaultRecordAndSampleResult = SamplingOverrides.getRecordAndSampleResult(defaultSamplingPercentage);
45+
recordAndSampleAndAddTraceStateIfMissing =
46+
SamplingOverrides.getRecordAndSampleAndAddTraceStateIfMissing(samplingPercentage);
4747

4848
this.samplingOverrides = samplingOverrides;
4949

@@ -53,7 +53,7 @@ class AiSampler implements Sampler {
5353
}
5454

5555
@Override
56-
public SamplingResult shouldSample(@Nullable Context parentContext,
56+
public SamplingResult shouldSample(Context parentContext,
5757
String traceId,
5858
String name,
5959
SpanKind spanKind,
@@ -63,23 +63,27 @@ public SamplingResult shouldSample(@Nullable Context parentContext,
6363
MatcherGroup override = samplingOverrides.getOverride(attributes);
6464

6565
if (override != null) {
66-
return getSamplingResult(override.getPercentage(), override.getRecordAndSampleResult(), traceId, name);
66+
return getSamplingResult(override.getPercentage(), override.getRecordAndSampleAndOverwriteTraceState(), traceId, name);
6767
}
6868

6969
switch (behaviorIfNoMatchingOverrides) {
7070
case RECORD_AND_SAMPLE:
71-
return defaultRecordAndSampleResult;
71+
// this is used for localParentSampled and remoteParentSampled
72+
// (note: currently sampling percentage portion of trace state is not propagated,
73+
// so it will always be missing in the remoteParentSampled case)
74+
return recordAndSampleAndAddTraceStateIfMissing;
7275
case USE_DEFAULT_SAMPLING_PERCENTAGE:
73-
return getSamplingResult(defaultSamplingPercentage, defaultRecordAndSampleResult, traceId, name);
76+
// this is used for root sampler
77+
return getSamplingResult(defaultSamplingPercentage, recordAndSampleAndAddTraceStateIfMissing, traceId, name);
7478
default:
7579
throw new IllegalStateException("Unexpected BehaviorIfNoMatchingOverrides: " + behaviorIfNoMatchingOverrides);
7680
}
7781
}
7882

79-
private SamplingResult getSamplingResult(double percentage, SamplingResult recordAndSampleResult, String traceId, String name) {
83+
private SamplingResult getSamplingResult(double percentage, SamplingResult sampledSamplingResult, String traceId, String name) {
8084
if (percentage == 100) {
8185
// optimization, no need to calculate score in this case
82-
return recordAndSampleResult;
86+
return sampledSamplingResult;
8387
}
8488
if (percentage == 0) {
8589
// optimization, no need to calculate score in this case
@@ -89,7 +93,7 @@ private SamplingResult getSamplingResult(double percentage, SamplingResult recor
8993
logger.debug("Item {} sampled out", name);
9094
return dropDecision;
9195
}
92-
return recordAndSampleResult;
96+
return sampledSamplingResult;
9397
}
9498

9599
@Override

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@ public static Sampler getSampler(double samplingPercentage, Configuration config
1313
AiSampler.BehaviorIfNoMatchingOverrides.RECORD_AND_SAMPLE);
1414
// ignoreRemoteParentNotSampled is currently needed
1515
// because .NET SDK always propagates trace flags "00" (not sampled)
16+
// NOTE: once we start propagating sampling percentage over the wire, we can use that to know that we can
17+
// respect upstream decision for remoteParentNotSampled
1618
Sampler remoteParentNotSampled = config.preview.ignoreRemoteParentNotSampled ? rootSampler : Sampler.alwaysOff();
1719
return Sampler.parentBasedBuilder(rootSampler)
1820
.setRemoteParentNotSampled(remoteParentNotSampled)

‎agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java‎

Lines changed: 78 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
package com.microsoft.applicationinsights.agent.internal.sampling;
22

3+
import java.math.BigDecimal;
4+
import java.math.MathContext;
35
import java.util.ArrayList;
46
import java.util.List;
57
import java.util.function.Predicate;
@@ -11,6 +13,7 @@
1113
import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.SamplingOverrideAttribute;
1214
import io.opentelemetry.api.common.AttributeKey;
1315
import io.opentelemetry.api.common.Attributes;
16+
import io.opentelemetry.api.trace.TraceState;
1417
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
1518
import io.opentelemetry.sdk.trace.samplers.SamplingResult;
1619

@@ -35,37 +38,100 @@ MatcherGroup getOverride(Attributes attributes) {
3538
return null;
3639
}
3740

38-
static SamplingResult getRecordAndSampleResult(double percentage) {
39-
Attributes alwaysOnAttributes;
40-
if (percentage != 100) {
41-
alwaysOnAttributes = Attributes.of(Exporter.AI_SAMPLING_PERCENTAGE_KEY, percentage);
42-
} else {
43-
// the exporter assumes 100 when the AI_SAMPLING_PERCENTAGE_KEY attribute is not present
44-
alwaysOnAttributes = Attributes.empty();
41+
static SamplingResult getRecordAndSampleAndOverwriteTraceState(double samplingPercentage) {
42+
return new TraceStateUpdatingSamplingResult(SamplingDecision.RECORD_AND_SAMPLE, toRoundedString(samplingPercentage), true);
43+
}
44+
45+
static SamplingResult getRecordAndSampleAndAddTraceStateIfMissing(double samplingPercentage) {
46+
return new TraceStateUpdatingSamplingResult(SamplingDecision.RECORD_AND_SAMPLE, toRoundedString(samplingPercentage), false);
47+
}
48+
49+
// TODO write test for
50+
// * 33.33333333333
51+
// * 66.66666666666
52+
// * 1.123456
53+
// * 50.0
54+
// * 1.0
55+
// * 0
56+
// * 0.001
57+
// * 0.000001
58+
// 5 digit of precision, and remove any trailing zeros beyond the decimal point
59+
private static String toRoundedString(double percentage) {
60+
BigDecimal bigDecimal = new BigDecimal(percentage);
61+
bigDecimal = bigDecimal.round(new MathContext(5));
62+
String formatted = bigDecimal.toString();
63+
double dv = bigDecimal.doubleValue();
64+
if (dv > 0 && dv < 1) {
65+
while (formatted.endsWith("0")) {
66+
formatted = formatted.substring(0, formatted.length() - 1);
67+
}
68+
}
69+
return formatted;
70+
}
71+
72+
private static final class TraceStateUpdatingSamplingResult implements SamplingResult {
73+
74+
private final SamplingDecision decision;
75+
private final String samplingPercentage;
76+
private final TraceState traceState;
77+
private final boolean overwriteExisting;
78+
79+
private TraceStateUpdatingSamplingResult(SamplingDecision decision, String samplingPercentage,
80+
boolean overwriteExisting) {
81+
this.decision = decision;
82+
this.samplingPercentage = samplingPercentage;
83+
this.overwriteExisting = overwriteExisting;
84+
traceState = TraceState.builder().put(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE, samplingPercentage).build();
85+
}
86+
87+
@Override
88+
public SamplingDecision getDecision() {
89+
return decision;
90+
}
91+
92+
@Override
93+
public Attributes getAttributes() {
94+
return Attributes.empty();
95+
}
96+
97+
@Override
98+
public TraceState getUpdatedTraceState(TraceState parentTraceState) {
99+
if (parentTraceState.isEmpty()) {
100+
return traceState;
101+
}
102+
String existingSamplingPercentage = parentTraceState.get(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE);
103+
if (samplingPercentage.equals(existingSamplingPercentage)) {
104+
return parentTraceState;
105+
}
106+
if (existingSamplingPercentage != null && !overwriteExisting) {
107+
return parentTraceState;
108+
}
109+
return parentTraceState.toBuilder()
110+
.put(Exporter.SAMPLING_PERCENTAGE_TRACE_STATE, samplingPercentage)
111+
.build();
45112
}
46-
return SamplingResult.create(SamplingDecision.RECORD_AND_SAMPLE, alwaysOnAttributes);
47113
}
48114

49115
static class MatcherGroup {
50116
private final List<Predicate<Attributes>> predicates;
51117
private final double percentage;
52-
private final SamplingResult recordAndSampleResult;
118+
private final SamplingResult recordAndSampleAndOverwriteTraceState;
53119

54120
private MatcherGroup(SamplingOverride override) {
55121
predicates = new ArrayList<>();
56122
for (SamplingOverrideAttribute attribute : override.attributes) {
57123
predicates.add(toPredicate(attribute));
58124
}
59125
percentage = override.percentage;
60-
recordAndSampleResult = SamplingOverrides.getRecordAndSampleResult(percentage);
126+
recordAndSampleAndOverwriteTraceState = SamplingOverrides.getRecordAndSampleAndOverwriteTraceState(percentage);
61127
}
62128

63129
double getPercentage() {
64130
return percentage;
65131
}
66132

67-
SamplingResult getRecordAndSampleResult() {
68-
return recordAndSampleResult;
133+
SamplingResult getRecordAndSampleAndOverwriteTraceState() {
134+
return recordAndSampleAndOverwriteTraceState;
69135
}
70136

71137
private boolean matches(Attributes attributes) {

0 commit comments

Comments
 (0)