Add telemetry filters (#1564)

* Add telemetry filtering

* Feedback

* Feedback

Author: trask

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/AiComponentInstaller.java

@@ -44,7 +44,6 @@
 import com.microsoft.applicationinsights.agent.internal.instrumentation.sdk.RequestTelemetryClassFileTransformer;
 import com.microsoft.applicationinsights.agent.internal.instrumentation.sdk.TelemetryClientClassFileTransformer;
 import com.microsoft.applicationinsights.agent.internal.instrumentation.sdk.WebRequestTrackingFilterClassFileTransformer;
-import com.microsoft.applicationinsights.agent.internal.sampling.SamplingPercentage;
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.RpConfiguration;
 import com.microsoft.applicationinsights.common.CommonUtils;
 import com.microsoft.applicationinsights.customExceptions.FriendlyException;
@@ -152,7 +151,7 @@ private static void start(Instrumentation instrumentation) {
         configuration.getContextInitializers().add(new SdkVersionContextInitializer());
         configuration.getContextInitializers().add(new ResourceAttributesContextInitializer(config.customDimensions));
 
-        Global.setSamplingPercentage(SamplingPercentage.roundToNearest(config.sampling.percentage));
+        Global.setSamplingPercentage(config.sampling.percentage);
         final TelemetryClient telemetryClient = new TelemetryClient();
         Global.setTelemetryClient(telemetryClient);
 
@@ -181,7 +180,7 @@ public void run() {
 
         RpConfiguration rpConfiguration = MainEntryPoint.getRpConfiguration();
         if (rpConfiguration != null) {
-            RpConfigurationPolling.startPolling(rpConfiguration);
+            RpConfigurationPolling.startPolling(rpConfiguration, config.preview.sampling.overrides);
         }
     }
 

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/RpConfigurationPolling.java

@@ -25,12 +25,14 @@
 import java.nio.file.Files;
 import java.nio.file.attribute.BasicFileAttributes;
 import java.nio.file.attribute.FileTime;
+import java.util.List;
 import java.util.concurrent.Executors;
 
 import com.microsoft.applicationinsights.TelemetryConfiguration;
 import com.microsoft.applicationinsights.agent.internal.sampling.DelegatingSampler;
 import com.microsoft.applicationinsights.agent.internal.sampling.Samplers;
-import com.microsoft.applicationinsights.agent.internal.sampling.SamplingPercentage;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.SamplingOverride;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.ConfigurationBuilder;
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.RpConfiguration;
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.RpConfigurationBuilder;
 import com.microsoft.applicationinsights.internal.util.ThreadPoolUtils;
@@ -41,17 +43,20 @@
 
 public class RpConfigurationPolling implements Runnable {
 
-    private volatile RpConfiguration rpConfiguration;
     private static final Logger logger = LoggerFactory.getLogger(RpConfigurationPolling.class);
 
-    // visible for testing
-    RpConfigurationPolling(RpConfiguration rpConfiguration) {
-        this.rpConfiguration = rpConfiguration;
-    }
+    private volatile RpConfiguration rpConfiguration;
+    private final List<SamplingOverride> samplingOverrides;
 
-    public static void startPolling(RpConfiguration rpConfiguration) {
+    public static void startPolling(RpConfiguration rpConfiguration, List<SamplingOverride> samplingOverrides) {
         Executors.newSingleThreadScheduledExecutor(ThreadPoolUtils.createDaemonThreadFactory(RpConfigurationPolling.class))
-                .scheduleWithFixedDelay(new RpConfigurationPolling(rpConfiguration), 60, 60, SECONDS);
+                .scheduleWithFixedDelay(new RpConfigurationPolling(rpConfiguration, samplingOverrides), 60, 60, SECONDS);
+    }
+
+    // visible for testing
+    RpConfigurationPolling(RpConfiguration rpConfiguration, List<SamplingOverride> samplingOverrides) {
+        this.rpConfiguration = rpConfiguration;
+        this.samplingOverrides = samplingOverrides;
     }
 
     @Override
@@ -79,8 +84,8 @@ public void run() {
 
                 if (newRpConfiguration.sampling.percentage != rpConfiguration.sampling.percentage) {
                     logger.debug("Updating sampling percentage from {} to {}", rpConfiguration.sampling.percentage, newRpConfiguration.sampling.percentage);
-                    double roundedSamplingPercentage = SamplingPercentage.roundToNearest(newRpConfiguration.sampling.percentage);
-                    DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(roundedSamplingPercentage));
+                    double roundedSamplingPercentage = ConfigurationBuilder.roundToNearest(newRpConfiguration.sampling.percentage);
+                    DelegatingSampler.getInstance().setDelegate(Samplers.getSampler(roundedSamplingPercentage, samplingOverrides));
                     Global.setSamplingPercentage(roundedSamplingPercentage);
                     rpConfiguration.sampling.percentage = newRpConfiguration.sampling.percentage;
                 }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/AgentProcessor.java

@@ -9,7 +9,7 @@
 
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorAttribute;
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorIncludeExclude;
-import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorMatchType;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.MatchType;
 import io.opentelemetry.api.common.AttributeKey;
 import io.opentelemetry.sdk.trace.data.SpanData;
 import org.checkerframework.checker.nullness.qual.Nullable;
@@ -25,7 +25,7 @@ public AgentProcessor(@Nullable IncludeExclude include,
     }
 
     protected static AttributeProcessor.IncludeExclude getNormalizedIncludeExclude(ProcessorIncludeExclude includeExclude) {
-        return includeExclude.matchType == ProcessorMatchType.strict ? AgentProcessor.StrictIncludeExclude.create(includeExclude) : AgentProcessor.RegexpIncludeExclude.create(includeExclude);
+        return includeExclude.matchType == MatchType.strict ? AgentProcessor.StrictIncludeExclude.create(includeExclude) : AgentProcessor.RegexpIncludeExclude.create(includeExclude);
     }
 
     public @Nullable IncludeExclude getInclude() {

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/AiSampler.java

@@ -3,7 +3,7 @@
 import java.util.List;
 import javax.annotation.Nullable;
 
-import com.microsoft.applicationinsights.agent.Exporter;
+import com.microsoft.applicationinsights.agent.internal.sampling.SamplingOverrides.MatcherGroup;
 import io.opentelemetry.api.common.Attributes;
 import io.opentelemetry.api.trace.SpanKind;
 import io.opentelemetry.context.Context;
@@ -18,29 +18,28 @@
 // * implements same trace id hashing algorithm so that traces are sampled the same across multiple nodes
 //   when some of those nodes are being monitored by other Application Insights SDKs (and 2.x Java SDK)
 // * adds sampling percentage to span attribute (TODO this is not being applied to child spans)
-public final class AiSampler implements Sampler {
+class AiSampler implements Sampler {
 
     private static final Logger logger = LoggerFactory.getLogger(AiSampler.class);
 
     // all sampling percentage must be in a ratio of 100/N where N is a whole number (2, 3, 4, ...)
     // e.g. 50 for 1/2 or 33.33 for 1/3
     //
     // failure to follow this pattern can result in unexpected / incorrect computation of values in the portal
-    private final double samplingPercentage;
+    private final double defaultSamplingPercentage;
+    private final SamplingResult defaultRecordAndSampleResult;
 
-    private final SamplingResult alwaysOnDecision;
-    private final SamplingResult alwaysOffDecision;
+    private final SamplingOverrides samplingOverrides;
 
-    public AiSampler(double samplingPercentage) {
-        this.samplingPercentage = samplingPercentage;
-        Attributes alwaysOnAttributes;
-        if (samplingPercentage != 100) {
-            alwaysOnAttributes = Attributes.of(Exporter.AI_SAMPLING_PERCENTAGE_KEY, samplingPercentage);
-        } else {
-            alwaysOnAttributes = Attributes.empty();
-        }
-        alwaysOnDecision = new FixedRateSamplerDecision(SamplingDecision.RECORD_AND_SAMPLE, alwaysOnAttributes);
-        alwaysOffDecision = new FixedRateSamplerDecision(SamplingDecision.DROP, Attributes.empty());
+    private final SamplingResult dropDecision;
+
+    AiSampler(double samplingPercentage, SamplingOverrides samplingOverrides) {
+        this.defaultSamplingPercentage = samplingPercentage;
+        defaultRecordAndSampleResult = SamplingOverrides.getRecordAndSampleResult(defaultSamplingPercentage);
+
+        this.samplingOverrides = samplingOverrides;
+
+        dropDecision = SamplingResult.create(SamplingDecision.DROP, Attributes.empty());
     }
 
     @Override
@@ -50,39 +49,37 @@ public SamplingResult shouldSample(@Nullable Context parentContext,
                                        SpanKind spanKind,
                                        Attributes attributes,
                                        List<LinkData> parentLinks) {
-        if (samplingPercentage == 100) {
-            return alwaysOnDecision;
+
+        MatcherGroup override = samplingOverrides.getOverride(attributes);
+
+        double percentage;
+        SamplingResult recordAndSampleResult;
+        if (override != null) {
+            percentage = override.getPercentage();
+            recordAndSampleResult = override.getRecordAndSampleResult();
+        } else {
+            // no overrides, so fall back to the default sampling percentage
+            percentage = defaultSamplingPercentage;
+            recordAndSampleResult = defaultRecordAndSampleResult;
+        }
+
+        if (percentage == 100) {
+            // optimization, no need to calculate score in this case
+            return recordAndSampleResult;
         }
-        if (SamplingScoreGeneratorV2.getSamplingScore(traceId) >= samplingPercentage) {
+        if (percentage == 0) {
+            // optimization, no need to calculate score in this case
+            return dropDecision;
+        }
+        if (SamplingScoreGeneratorV2.getSamplingScore(traceId) >= percentage) {
             logger.debug("Item {} sampled out", name);
-            return alwaysOffDecision;
+            return dropDecision;
         }
-        return alwaysOnDecision;
+        return recordAndSampleResult;
     }
 
     @Override
     public String getDescription() {
-        return "ApplicationInsights-specific trace id based sampler, with sampling percentage: " + samplingPercentage;
-    }
-
-    private static final class FixedRateSamplerDecision implements SamplingResult {
-
-        private final SamplingDecision decision;
-        private final Attributes attributes;
-
-        private FixedRateSamplerDecision(SamplingDecision decision, Attributes attributes) {
-            this.decision = decision;
-            this.attributes = attributes;
-        }
-
-        @Override
-        public SamplingDecision getDecision() {
-            return decision;
-        }
-
-        @Override
-        public Attributes getAttributes() {
-            return attributes;
-        }
+        return "ApplicationInsights-specific trace id based sampler, with default sampling percentage: " + defaultSamplingPercentage;
     }
 }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/Samplers.java

@@ -1,22 +1,29 @@
 package com.microsoft.applicationinsights.agent.internal.sampling;
 
+import java.util.List;
+
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.SamplingOverride;
 import io.opentelemetry.sdk.trace.samplers.Sampler;
 
 public class Samplers {
 
-    public static Sampler getSampler(double samplingPercentage) {
-        if (samplingPercentage != 100) {
-            AiSampler aiSampler = new AiSampler(samplingPercentage);
-            return Sampler.parentBasedBuilder(aiSampler)
-                    // for now, we have to override default behavior for "not sampled" remote parents
-                    // because .NET SDK always propagates trace flags "00" (not sampled)
-                    .setRemoteParentNotSampled(aiSampler)
-                    .build();
-        } else {
-            // OpenTelemetry default sampling is "parent based", which means don't sample if remote traceparent sampled flag was not set,
-            // but Application Insights SDKs do not send the sampled flag (since they perform sampling during export instead of head-based sampling)
-            // so need to use "always on" in this case
-            return io.opentelemetry.sdk.trace.samplers.Sampler.alwaysOn();
-        }
+    public static Sampler getSampler(double samplingPercentage, List<SamplingOverride> overrides) {
+        SamplingOverrides samplingOverrides = new SamplingOverrides(overrides);
+        AiSampler aiSampler = new AiSampler(samplingPercentage, samplingOverrides);
+        return Sampler.parentBasedBuilder(aiSampler)
+                // TODO change the default behavior, and provide preview flag to treat "00" as
+                // for now, we have to override default behavior of "alwaysOff"
+                // because .NET SDK always propagates trace flags "00" (not sampled)
+                .setRemoteParentNotSampled(aiSampler)
+                // this is the default, just including it for completeness
+                // intentionally not allowing to capture a downstream span when upstream span has not been sampled
+                // because this will lead to broken traces in A (sampled) -> B (not sampled) -> C (sampled)
+                // C will point to parent B, but B will not be exported
+                .setLocalParentNotSampled(Sampler.alwaysOff())
+                // can filter out subtree of sampled trace, by applying sampling override
+                .setRemoteParentSampled(aiSampler)
+                // can filter out subtree of sampled trace, by applying sampling override
+                .setLocalParentSampled(aiSampler)
+                .build();
     }
 }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/sampling/SamplingOverrides.java

@@ -0,0 +1,122 @@
+package com.microsoft.applicationinsights.agent.internal.sampling;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.regex.Pattern;
+
+import com.microsoft.applicationinsights.agent.Exporter;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.MatchType;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.SamplingOverride;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.SamplingOverrideAttribute;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.sdk.trace.samplers.SamplingDecision;
+import io.opentelemetry.sdk.trace.samplers.SamplingResult;
+
+// TODO find a better name for this class (and MatcherGroup too)
+class SamplingOverrides {
+
+    private final List<MatcherGroup> matcherGroups;
+
+    SamplingOverrides(List<SamplingOverride> overrides) {
+        matcherGroups = new ArrayList<>();
+        for (SamplingOverride override : overrides) {
+            matcherGroups.add(new MatcherGroup(override));
+        }
+    }
+
+    MatcherGroup getOverride(Attributes attributes) {
+        for (MatcherGroup matcherGroups : matcherGroups) {
+            if (matcherGroups.matches(attributes)) {
+                return matcherGroups;
+            }
+        }
+        return null;
+    }
+
+    static SamplingResult getRecordAndSampleResult(double percentage) {
+        Attributes alwaysOnAttributes;
+        if (percentage != 100) {
+            alwaysOnAttributes = Attributes.of(Exporter.AI_SAMPLING_PERCENTAGE_KEY, percentage);
+        } else {
+            // the exporter assumes 100 when the AI_SAMPLING_PERCENTAGE_KEY attribute is not present
+            alwaysOnAttributes = Attributes.empty();
+        }
+        return SamplingResult.create(SamplingDecision.RECORD_AND_SAMPLE, alwaysOnAttributes);
+    }
+
+    static class MatcherGroup {
+        private final List<Predicate<Attributes>> predicates;
+        private final double percentage;
+        private final SamplingResult recordAndSampleResult;
+
+        private MatcherGroup(SamplingOverride override) {
+            predicates = new ArrayList<>();
+            for (SamplingOverrideAttribute attribute : override.attributes) {
+                predicates.add(toPredicate(attribute));
+            }
+            percentage = override.percentage;
+            recordAndSampleResult = SamplingOverrides.getRecordAndSampleResult(percentage);
+        }
+
+        double getPercentage() {
+            return percentage;
+        }
+
+        SamplingResult getRecordAndSampleResult() {
+            return recordAndSampleResult;
+        }
+
+        private boolean matches(Attributes attributes) {
+            for (Predicate<Attributes> predicate : predicates) {
+                if (!predicate.test(attributes)) {
+                    return false;
+                }
+            }
+            return true;
+        }
+
+        private static Predicate<Attributes> toPredicate(SamplingOverrideAttribute attribute) {
+            if (attribute.matchType == MatchType.strict) {
+                return new StrictMatcher(attribute.key, attribute.value);
+            } else if (attribute.matchType == MatchType.regexp) {
+                return new RegexpMatcher(attribute.key, attribute.value);
+            } else {
+                throw new IllegalStateException("Unexpected match type: " + attribute.matchType);
+            }
+        }
+    }
+
+    private static class StrictMatcher implements Predicate<Attributes> {
+        private final AttributeKey<String> key;
+        private final String value;
+
+        private StrictMatcher(String key, String value) {
+            this.key = AttributeKey.stringKey(key);
+            this.value = value;
+        }
+
+        @Override
+        public boolean test(Attributes attributes) {
+            String val = attributes.get(key);
+            return value.equals(val);
+        }
+    }
+
+    private static class RegexpMatcher implements Predicate<Attributes> {
+        private final AttributeKey<String> key;
+        private final Pattern value;
+
+        private RegexpMatcher(String key, String value) {
+            this.key = AttributeKey.stringKey(key);
+            this.value = Pattern.compile(value);
+        }
+
+        @Override
+        public boolean test(Attributes attributes) {
+            String val = attributes.get(key);
+            return val != null && value.matcher(val).matches();
+        }
+    }
+}