Skip to content

Commit 59be9a0

Browse files
committed
Improve sampling override behavior
1 parent da24299 commit 59be9a0

File tree

7 files changed

+234
-56
lines changed

7 files changed

+234
-56
lines changed

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/exporter/AgentLogExporter.java

Lines changed: 18 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,14 @@
44
package com.microsoft.applicationinsights.agent.internal.exporter;
55

66
import static com.azure.monitor.opentelemetry.exporter.implementation.utils.AzureMonitorMsgId.EXPORTER_MAPPING_ERROR;
7-
import static com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils.shouldSample;
87

8+
import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes;
99
import com.azure.monitor.opentelemetry.exporter.implementation.LogDataMapper;
1010
import com.azure.monitor.opentelemetry.exporter.implementation.logging.OperationLogger;
1111
import com.azure.monitor.opentelemetry.exporter.implementation.models.TelemetryItem;
1212
import com.azure.monitor.opentelemetry.exporter.implementation.quickpulse.QuickPulse;
1313
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverride;
14+
import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
1415
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
1516
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
1617
import com.microsoft.applicationinsights.agent.internal.telemetry.TelemetryClient;
@@ -21,6 +22,7 @@
2122
import io.opentelemetry.sdk.common.CompletableResultCode;
2223
import io.opentelemetry.sdk.logs.data.LogRecordData;
2324
import io.opentelemetry.sdk.logs.export.LogRecordExporter;
25+
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
2426
import io.opentelemetry.semconv.SemanticAttributes;
2527
import java.util.Collection;
2628
import java.util.List;
@@ -108,24 +110,30 @@ private CompletableResultCode internalExport(Collection<LogRecordData> logs) {
108110
stack != null ? exceptionSamplingOverrides : logSamplingOverrides;
109111

110112
SpanContext spanContext = log.getSpanContext();
113+
Double parentSpanSampleRate = log.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE);
111114

112-
Double samplingPercentage = samplingOverrides.getOverridePercentage(log.getAttributes());
115+
AiSampler sampler = samplingOverrides.getOverride(log.getAttributes());
113116

114-
if (samplingPercentage != null && !shouldSample(spanContext, samplingPercentage)) {
115-
continue;
116-
}
117-
118-
if (samplingPercentage == null
119-
&& spanContext.isValid()
120-
&& !spanContext.getTraceFlags().isSampled()) {
117+
if (sampler == null && spanContext.isValid() && !spanContext.getTraceFlags().isSampled()) {
121118
// if there is no sampling override, and the log is part of an unsampled trace, then don't
122119
// capture it
123120
continue;
124121
}
125122

123+
Double sampleRate = parentSpanSampleRate;
124+
if (sampler != null) {
125+
if (sampler.shouldSampleLog(spanContext, parentSpanSampleRate).getDecision()
126+
== SamplingDecision.DROP) {
127+
continue;
128+
}
129+
// sampling override percentage takes precedence
130+
sampleRate = sampler.getParentlessDependencySamplingPercentage().get();
131+
}
132+
126133
logger.debug("exporting log: {}", log);
127134

128-
TelemetryItem telemetryItem = mapper.map(log, stack, samplingPercentage);
135+
// TODO (trask) no longer need to check AiSemanticAttributes.SAMPLE_RATE in map() method
136+
TelemetryItem telemetryItem = mapper.map(log, stack, sampleRate);
129137
telemetryItemConsumer.accept(telemetryItem);
130138

131139
exportingLogLogger.recordSuccess();

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/init/SecondEntryPoint.java

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66
import static java.util.concurrent.TimeUnit.MINUTES;
77

88
import com.azure.core.util.logging.ClientLogger;
9+
import com.azure.monitor.opentelemetry.exporter.implementation.AiSemanticAttributes;
910
import com.azure.monitor.opentelemetry.exporter.implementation.AzureMonitorExporterProviderKeys;
1011
import com.azure.monitor.opentelemetry.exporter.implementation.AzureMonitorLogRecordExporterProvider;
1112
import com.azure.monitor.opentelemetry.exporter.implementation.AzureMonitorMetricExporterProvider;
@@ -37,14 +38,14 @@
3738
import com.microsoft.applicationinsights.agent.internal.exporter.AgentLogExporter;
3839
import com.microsoft.applicationinsights.agent.internal.exporter.AgentMetricExporter;
3940
import com.microsoft.applicationinsights.agent.internal.exporter.AgentSpanExporter;
40-
import com.microsoft.applicationinsights.agent.internal.exporter.ExporterUtils;
4141
import com.microsoft.applicationinsights.agent.internal.httpclient.LazyHttpClient;
4242
import com.microsoft.applicationinsights.agent.internal.legacyheaders.AiLegacyHeaderSpanProcessor;
4343
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithLogProcessor;
4444
import com.microsoft.applicationinsights.agent.internal.processors.ExporterWithSpanProcessor;
4545
import com.microsoft.applicationinsights.agent.internal.processors.LogExporterWithAttributeProcessor;
4646
import com.microsoft.applicationinsights.agent.internal.processors.SpanExporterWithAttributeProcessor;
4747
import com.microsoft.applicationinsights.agent.internal.profiler.triggers.AlertTriggerSpanProcessor;
48+
import com.microsoft.applicationinsights.agent.internal.sampling.AiSampler;
4849
import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides;
4950
import com.microsoft.applicationinsights.agent.internal.telemetry.BatchItemProcessor;
5051
import com.microsoft.applicationinsights.agent.internal.telemetry.MetricFilter;
@@ -66,6 +67,7 @@
6667
import io.opentelemetry.sdk.metrics.internal.view.AiViewRegistry;
6768
import io.opentelemetry.sdk.trace.SdkTracerProviderBuilder;
6869
import io.opentelemetry.sdk.trace.export.SpanExporter;
70+
import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
6971
import java.io.File;
7072
import java.util.ArrayList;
7173
import java.util.Collections;
@@ -579,10 +581,14 @@ private static SpanExporter createSpanExporter(
579581
return false;
580582
},
581583
(span, event) -> {
582-
Double samplingPercentage =
583-
exceptionSamplingOverrides.getOverridePercentage(event.getAttributes());
584-
return samplingPercentage != null
585-
&& !ExporterUtils.shouldSample(span.getSpanContext(), samplingPercentage);
584+
AiSampler sampler = exceptionSamplingOverrides.getOverride(event.getAttributes());
585+
return sampler != null
586+
&& sampler
587+
.shouldSampleLog(
588+
span.getSpanContext(),
589+
span.getAttributes().get(AiSemanticAttributes.SAMPLE_RATE))
590+
.getDecision()
591+
== SamplingDecision.DROP;
586592
});
587593

588594
BatchItemProcessor batchItemProcessor = telemetryClient.getGeneralBatchItemProcessor();

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java

Lines changed: 68 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -30,32 +30,48 @@ public class AiSampler implements Sampler {
3030
private static final double SAMPLE_RATE_TO_DISABLE_INGESTION_SAMPLING = 99.99;
3131

3232
private final boolean ingestionSamplingEnabled;
33-
private final boolean localParentBased;
33+
private final boolean sampleWhenLocalParentSampled;
34+
private final boolean dropWhenLocalParentDropped;
3435
private final SamplingPercentage requestSamplingPercentage;
3536
// when localParentBased=false, then this applies to all dependencies, not only parentless
3637
private final SamplingPercentage parentlessDependencySamplingPercentage;
3738
private final Cache<Double, SamplingResult> recordAndSampleWithSampleRateMap = Cache.bounded(100);
3839

39-
public AiSampler(
40+
public static AiSampler create(
4041
SamplingPercentage requestSamplingPercentage,
4142
SamplingPercentage parentlessDependencySamplingPercentage,
4243
boolean ingestionSamplingEnabled) {
43-
this(
44+
return new AiSampler(
4445
requestSamplingPercentage,
4546
parentlessDependencySamplingPercentage,
4647
ingestionSamplingEnabled,
48+
true,
4749
true);
4850
}
4951

50-
public AiSampler(
52+
public static AiSampler createSamplingOverride(
53+
SamplingPercentage samplingPercentage,
54+
boolean sampleWhenLocalParentSampled,
55+
boolean dropWhenLocalParentDropped) {
56+
return new AiSampler(
57+
samplingPercentage,
58+
samplingPercentage,
59+
false,
60+
sampleWhenLocalParentSampled,
61+
dropWhenLocalParentDropped);
62+
}
63+
64+
private AiSampler(
5165
SamplingPercentage requestSamplingPercentage,
5266
SamplingPercentage parentlessDependencySamplingPercentage,
5367
boolean ingestionSamplingEnabled,
54-
boolean localParentBased) {
68+
boolean sampleWhenLocalParentSampled,
69+
boolean dropWhenLocalParentDropped) {
5570
this.requestSamplingPercentage = requestSamplingPercentage;
5671
this.parentlessDependencySamplingPercentage = parentlessDependencySamplingPercentage;
5772
this.ingestionSamplingEnabled = ingestionSamplingEnabled;
58-
this.localParentBased = localParentBased;
73+
this.sampleWhenLocalParentSampled = sampleWhenLocalParentSampled;
74+
this.dropWhenLocalParentDropped = dropWhenLocalParentDropped;
5975
}
6076

6177
@Override
@@ -66,8 +82,42 @@ public SamplingResult shouldSample(
6682
SpanKind spanKind,
6783
Attributes attributes,
6884
List<LinkData> parentLinks) {
69-
if (localParentBased) {
70-
SamplingResult samplingResult = useLocalParentDecisionIfPossible(parentContext);
85+
86+
Span parentSpan = Span.fromContext(parentContext);
87+
SpanContext parentSpanContext = parentSpan.getSpanContext();
88+
Double parentSpanSampleRate = null;
89+
if (parentSpan instanceof ReadableSpan) {
90+
parentSpanSampleRate =
91+
((ReadableSpan) parentSpan).getAttribute(AiSemanticAttributes.SAMPLE_RATE);
92+
}
93+
94+
return internalShouldSample(
95+
parentSpanContext, parentSpanSampleRate, traceId, spanKind, attributes);
96+
}
97+
98+
public SamplingResult shouldSampleLog(SpanContext spanContext, @Nullable Double spanSampleRate) {
99+
return internalShouldSample(
100+
spanContext,
101+
spanSampleRate,
102+
spanContext.getTraceId(),
103+
SpanKind.INTERNAL, // unused
104+
Attributes.empty());
105+
}
106+
107+
public SamplingPercentage getParentlessDependencySamplingPercentage() {
108+
return parentlessDependencySamplingPercentage;
109+
}
110+
111+
private SamplingResult internalShouldSample(
112+
SpanContext parentSpanContext,
113+
@Nullable Double parentSpanSampleRate,
114+
String traceId,
115+
SpanKind spanKind,
116+
Attributes attributes) {
117+
118+
if (sampleWhenLocalParentSampled || dropWhenLocalParentDropped) {
119+
SamplingResult samplingResult =
120+
useLocalParentDecisionIfPossible(parentSpanContext, parentSpanSampleRate);
71121
if (samplingResult != null) {
72122
return samplingResult;
73123
}
@@ -78,7 +128,6 @@ public SamplingResult shouldSample(
78128
// optimization for fixed-rate sampling
79129
sp = requestSamplingPercentage.get();
80130
} else {
81-
SpanContext parentSpanContext = Span.fromContext(parentContext).getSpanContext();
82131
boolean isRequest = RequestChecker.isRequest(spanKind, parentSpanContext, attributes::get);
83132
sp =
84133
isRequest
@@ -113,26 +162,26 @@ public SamplingResult shouldSample(
113162
return samplingResult;
114163
}
115164

165+
@SuppressWarnings("SystemOut")
116166
@Nullable
117-
private static SamplingResult useLocalParentDecisionIfPossible(Context parentContext) {
167+
private SamplingResult useLocalParentDecisionIfPossible(
168+
SpanContext parentSpanContext, @Nullable Double parentSpanSampleRate) {
118169
// remote parent-based sampling messes up item counts since item count is not propagated in
119170
// tracestate (yet), but local parent-based sampling doesn't have this issue since we are
120171
// propagating item count locally
121-
Span parentSpan = Span.fromContext(parentContext);
122-
SpanContext parentSpanContext = parentSpan.getSpanContext();
123172
if (!parentSpanContext.isValid() || parentSpanContext.isRemote()) {
173+
System.out.println("useLocalParentDecisionIfPossible1");
124174
return null;
125175
}
126176
if (!parentSpanContext.isSampled()) {
127-
return SamplingResult.drop();
177+
System.out.println("useLocalParentDecisionIfPossible2: " + dropWhenLocalParentDropped);
178+
return dropWhenLocalParentDropped ? SamplingResult.drop() : null;
128179
}
129-
if (parentSpan instanceof ReadableSpan) {
130-
Double parentSampleRate =
131-
((ReadableSpan) parentSpan).getAttribute(AiSemanticAttributes.SAMPLE_RATE);
132-
if (parentSampleRate != null) {
133-
return new RecordAndSampleWithItemCount(parentSampleRate);
134-
}
180+
if (sampleWhenLocalParentSampled && parentSpanSampleRate != null) {
181+
System.out.println("useLocalParentDecisionIfPossible3");
182+
return new RecordAndSampleWithItemCount(parentSpanSampleRate);
135183
}
184+
System.out.println("useLocalParentDecisionIfPossible5");
136185
return null;
137186
}
138187

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,14 +19,14 @@ public static Sampler getSampler(
1919
SamplingPercentage.rateLimited(sampling.requestsPerSecond);
2020
SamplingPercentage parentlessDependencySamplingPercentage = SamplingPercentage.fixed(100);
2121
sampler =
22-
new AiSampler(
22+
AiSampler.create(
2323
requestSamplingPercentage,
2424
parentlessDependencySamplingPercentage,
2525
samplingPreview.ingestionSamplingEnabled);
2626
} else if (sampling.percentage != null) {
2727
SamplingPercentage samplingPercentage = SamplingPercentage.fixed(sampling.percentage);
2828
sampler =
29-
new AiSampler(
29+
AiSampler.create(
3030
samplingPercentage, samplingPercentage, samplingPreview.ingestionSamplingEnabled);
3131
} else {
3232
throw new AssertionError("ConfigurationBuilder should have set the default sampling");

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java

Lines changed: 9 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,6 @@
99
import com.microsoft.applicationinsights.agent.internal.configuration.Configuration.SamplingOverrideAttribute;
1010
import io.opentelemetry.api.common.AttributeKey;
1111
import io.opentelemetry.api.common.Attributes;
12-
import io.opentelemetry.sdk.trace.samplers.Sampler;
1312
import io.opentelemetry.semconv.SemanticAttributes;
1413
import java.util.ArrayList;
1514
import java.util.List;
@@ -32,7 +31,7 @@ public SamplingOverrides(List<SamplingOverride> overrides) {
3231
}
3332

3433
@Nullable
35-
public Sampler getOverride(Attributes attributes) {
34+
public AiSampler getOverride(Attributes attributes) {
3635
LazyHttpUrl lazyHttpUrl = new LazyHttpUrl(attributes);
3736
LazyHttpTarget lazyHttpTarget = new LazyHttpTarget(attributes);
3837
for (MatcherGroup matcherGroups : matcherGroups) {
@@ -43,20 +42,9 @@ public Sampler getOverride(Attributes attributes) {
4342
return null;
4443
}
4544

46-
// used to do sampling inside the log exporter
47-
@Nullable
48-
public Double getOverridePercentage(Attributes attributes) {
49-
for (MatcherGroup matcherGroups : matcherGroups) {
50-
if (matcherGroups.matches(attributes, null, null)) {
51-
return matcherGroups.getPercentage();
52-
}
53-
}
54-
return null;
55-
}
56-
5745
private static class MatcherGroup {
5846
private final List<TempPredicate> predicates;
59-
private final Sampler sampler;
47+
private final AiSampler sampler;
6048
// for now only support fixed percentage, but could extend sampling overrides to support
6149
// rate-limited sampling
6250
private final SamplingPercentage samplingPercentage;
@@ -70,17 +58,18 @@ private MatcherGroup(SamplingOverride override) {
7058
}
7159
}
7260
samplingPercentage = SamplingPercentage.fixed(override.percentage);
73-
sampler = new AiSampler(samplingPercentage, samplingPercentage, false, false);
61+
// setting sampleWhenLocalParentSampled = (override.percentage == 100) would end up the same
62+
boolean sampleWhenLocalParentSampled = false;
63+
boolean dropWhenLocalParentDropped = override.percentage < 100;
64+
sampler =
65+
AiSampler.createSamplingOverride(
66+
samplingPercentage, sampleWhenLocalParentSampled, dropWhenLocalParentDropped);
7467
}
7568

76-
Sampler getSampler() {
69+
AiSampler getSampler() {
7770
return sampler;
7871
}
7972

80-
double getPercentage() {
81-
return samplingPercentage.get();
82-
}
83-
8473
private boolean matches(
8574
Attributes attributes,
8675
@Nullable LazyHttpUrl lazyHttpUrl,

0 commit comments

Comments
 (0)