Skip to content

Commit 5fd738d

Browse files
dhaval24dhaval24
authored
Merge pull request #459 from Microsoft/Feature/RevampAdaptiveSampling
Introducing Fixed Rate Sampling v2 (Using Processors)
2 parents 0fc02a3 + ef17f19 commit 5fd738d

13 files changed

+1329
-1
lines changed

‎CHANGELOG.md‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,12 @@
11
# CHANGELOG
22

33
## Version 1.0.11
4+
- Introducing public method 'getIncludedTypes' and 'getExcludedTypes' in TelemetryProcessorXmlElement.
5+
- Introducing class 'com.microsoft.applicationinsights.internal.config.ParamIncludedTypeXmlElement'.
6+
- Introducing class 'com.microsoft.applicationinsights.internal.config.ParamExcludedTypeXmlElement'
7+
- Introducing class 'com.microsoft.applicationinsights.internal.channel.samplingV2.SamplingScoreGeneratorV2'
8+
- Introducing Telemetry Processor 'com.microsoft.applicationinsights.internal.channel.samplingV2.FixedRateSamplingTelemetryProcessor'
9+
- Introducing FixedRate Sampling v2 Using Telemetry Processors
410
- Fixed issue #436 (TraceTelemetry with Severity is not shown in UI). This fixes a regression issue with `TelemetryClient.trackTrace` and `TelemetryClient.trackException`.
511

612
## Version 1.0.10

‎core/src/main/java/com/microsoft/applicationinsights/internal/channel/samplingV2/FixedRateSamplingTelemetryProcessor.java‎

Lines changed: 217 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,217 @@
1+
package com.microsoft.applicationinsights.internal.channel.samplingV2;
2+
3+
import com.microsoft.applicationinsights.agent.internal.common.StringUtils;
4+
import com.microsoft.applicationinsights.extensibility.TelemetryProcessor;
5+
import com.microsoft.applicationinsights.internal.annotation.BuiltInProcessor;
6+
import com.microsoft.applicationinsights.internal.logger.InternalLogger;
7+
import com.microsoft.applicationinsights.telemetry.SupportSampling;
8+
import com.microsoft.applicationinsights.telemetry.Telemetry;
9+
10+
import java.util.HashMap;
11+
import java.util.HashSet;
12+
import java.util.Map;
13+
import java.util.Set;
14+
15+
/**
16+
* This processor is used to Perform Sampling on User specified sampling rate
17+
* <p>
18+
* How to use in ApplicationInsights Configuration :
19+
* <p>
20+
* <TelemetryProcessors>
21+
<BuiltInProcessors>
22+
<Processor type = "FixedRateSamplingTelemetryProcessor">
23+
<Add name = "SamplingPercentage" value = "50" />
24+
<ExcludedTypes>
25+
<ExcludedType>Request</ExcludedType>
26+
</ExcludedTypes>
27+
<IncludedTypes>
28+
<IncludedType>Request</IncludedType>
29+
<IncludedType>Trace</IncludedType>
30+
<IncludedType>Dependency</IncludedType>
31+
<IncludedType>Exception</IncludedType>
32+
</IncludedTypes>
33+
</Processor>
34+
</BuiltInProcessors>
35+
</TelemetryProcessors>
36+
*/
37+
@BuiltInProcessor("FixedRateSamplingTelemetryProcessor")
38+
public final class FixedRateSamplingTelemetryProcessor implements TelemetryProcessor {
39+
40+
private final String dependencyTelemetryName = "Dependency";
41+
private static final String eventTelemetryName = "Event";
42+
private static final String exceptionTelemetryName = "Exception";
43+
private static final String pageViewTelemetryName = "PageView";
44+
private static final String requestTelemetryName = "Request";
45+
private static final String traceTelemetryName = "Trace";
46+
47+
private static Map<String, Class> allowedTypes;
48+
49+
private Set<Class> excludedTypes;
50+
51+
private Set<Class> includedTypes;
52+
53+
/**
54+
* All sampling percentage must be in a ratio of 100/N where N is a whole number (2, 3, 4, …). E.g. 50 for 1/2 or 33.33 for 1/3.
55+
* Failure to follow this pattern can result in unexpected / incorrect computation of values in the portal.
56+
*/
57+
private double samplingPercentage;
58+
59+
/**
60+
* constructor is responsible of initializing this processor
61+
* to default settings
62+
*/
63+
public FixedRateSamplingTelemetryProcessor() {
64+
this.samplingPercentage = 100.00;
65+
this.includedTypes = new HashSet<Class>();
66+
this.excludedTypes = new HashSet<Class>();
67+
try {
68+
this.allowedTypes = new HashMap<String, Class>() {{
69+
put(dependencyTelemetryName, Class.forName("com.microsoft.applicationinsights.telemetry.RemoteDependencyTelemetry"));
70+
put(eventTelemetryName, Class.forName("com.microsoft.applicationinsights.telemetry.EventTelemetry"));
71+
put(exceptionTelemetryName, Class.forName("com.microsoft.applicationinsights.telemetry.ExceptionTelemetry"));
72+
put(pageViewTelemetryName, Class.forName("com.microsoft.applicationinsights.telemetry.PageViewTelemetry"));
73+
put(requestTelemetryName, Class.forName("com.microsoft.applicationinsights.telemetry.RequestTelemetry"));
74+
put(traceTelemetryName, Class.forName("com.microsoft.applicationinsights.telemetry.TraceTelemetry"));
75+
}};
76+
} catch (ClassNotFoundException e) {
77+
InternalLogger.INSTANCE.trace("Unable to locate telemetry classes");
78+
}
79+
}
80+
81+
/**
82+
* This method returns a set of classes of excluded types specified by user
83+
*
84+
* @return
85+
*/
86+
public Set<Class> getExcludedTypes() {
87+
return excludedTypes;
88+
}
89+
90+
/**
91+
* This method returns a set of classes of included types specified by user
92+
*
93+
* @return
94+
*/
95+
public Set<Class> getIncludedTypes() {
96+
return includedTypes;
97+
}
98+
99+
100+
private void setIncludedOrExcludedTypes(String value, Set<Class> typeSet) {
101+
102+
if (!StringUtils.isNullOrEmpty(value)) {
103+
value = value.trim();
104+
if (!StringUtils.isNullOrEmpty(value) && allowedTypes.containsKey(value)) {
105+
typeSet.add(allowedTypes.get(value));
106+
} else {
107+
InternalLogger.INSTANCE.error("Item is either not allowed to sample or is empty");
108+
}
109+
} else {
110+
InternalLogger.INSTANCE.error("Empty types cannot be considered");
111+
}
112+
}
113+
114+
/**
115+
* Gets the sample rate currently set
116+
*/
117+
double getSamplingPercentage() {
118+
return samplingPercentage;
119+
}
120+
121+
/**
122+
* Sets the user defined sampling percentage
123+
*
124+
* @param samplingPercentage
125+
*/
126+
public void setSamplingPercentage(String samplingPercentage) {
127+
try {
128+
this.samplingPercentage = Double.valueOf(samplingPercentage);
129+
InternalLogger.INSTANCE.info("Sampling rate set to " + samplingPercentage);
130+
}
131+
catch (NumberFormatException ex) {
132+
this.samplingPercentage = 100.0;
133+
InternalLogger.INSTANCE.error("Sampling rate specified in improper format, sampling rate is now set to 100.0 (default)");
134+
}
135+
}
136+
137+
/**
138+
* This method determines if the telemetry needs to be sampled or not.
139+
*
140+
* @param telemetry
141+
* @return
142+
*/
143+
@Override
144+
public boolean process(Telemetry telemetry) {
145+
146+
double samplingPercentage = this.samplingPercentage;
147+
148+
if (telemetry instanceof SupportSampling) {
149+
150+
if (isSamplingApplicable(telemetry.getClass())) {
151+
152+
SupportSampling samplingSupportingTelemetry = ((SupportSampling) telemetry);
153+
154+
if (samplingSupportingTelemetry.getSamplingPercentage() == null) {
155+
156+
samplingSupportingTelemetry.setSamplingPercentage(samplingPercentage);
157+
158+
if (SamplingScoreGeneratorV2.getSamplingScore(telemetry) >= samplingPercentage) {
159+
160+
InternalLogger.INSTANCE.info("Item %s sampled out", telemetry.getClass());
161+
return false;
162+
}
163+
} else {
164+
InternalLogger.INSTANCE.info("Item has sampling percentage already set to :"
165+
+ samplingSupportingTelemetry.getSamplingPercentage());
166+
}
167+
} else {
168+
InternalLogger.INSTANCE.trace("Skip sampling since %s type is not sampling applicable", telemetry.getClass());
169+
}
170+
}
171+
172+
return true;
173+
}
174+
175+
/**
176+
* Determines if the argument is applicable for sampling
177+
*
178+
* @param item : Denotes the class item to be determined applicable for sampling
179+
* @return boolean
180+
*/
181+
private boolean isSamplingApplicable(Class item) {
182+
183+
if (excludedTypes.size() > 0 && excludedTypes.contains(item)) {
184+
return false;
185+
}
186+
187+
if (includedTypes.size() > 0 && !includedTypes.contains(item)) {
188+
return false;
189+
}
190+
191+
return true;
192+
}
193+
194+
/**
195+
* This method is invoked during configuration to add one element to the
196+
* excluded types set from the xml array list of excluded types
197+
* @param value
198+
*/
199+
public void addToExcludedType(String value) {
200+
201+
setIncludedOrExcludedTypes(value, excludedTypes);
202+
InternalLogger.INSTANCE.trace(value + " added as excluded to sampling");
203+
204+
}
205+
206+
/**
207+
* This method is invoked during configuration to add one element to the
208+
* included types set from the xml array list of included types
209+
* @param value
210+
*/
211+
public void addToIncludedType(String value) {
212+
213+
setIncludedOrExcludedTypes(value, includedTypes);
214+
InternalLogger.INSTANCE.trace(value + " added as included to sampling");
215+
216+
}
217+
}

‎core/src/main/java/com/microsoft/applicationinsights/internal/channel/samplingV2/SamplingScoreGeneratorV2.java‎

Lines changed: 57 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,57 @@
1+
package com.microsoft.applicationinsights.internal.channel.samplingV2;
2+
3+
import com.microsoft.applicationinsights.agent.internal.common.StringUtils;
4+
import com.microsoft.applicationinsights.telemetry.Telemetry;
5+
6+
import java.util.Random;
7+
8+
/**
9+
* Created by Dhaval Doshi Oct 2017
10+
* This class generates the sample using the random number generator.
11+
* It also contains the logic to preserve the correlated telemetry items.
12+
*/
13+
public class SamplingScoreGeneratorV2 {
14+
15+
private static Random random = new Random();
16+
17+
/**
18+
* This method takes the telemetry and returns the hash of the operation id if it is present already
19+
* or uses the random number generator to generate the sampling score.
20+
* @param telemetry
21+
* @return
22+
*/
23+
public static double getSamplingScore(Telemetry telemetry) {
24+
25+
double samplingScore = 0.0;
26+
27+
if (!StringUtils.isNullOrEmpty(telemetry.getContext().getOperation().getId())) {
28+
samplingScore = ((double) getSamplingHashCode(telemetry.getContext().getOperation().getId()) / Integer.MAX_VALUE);
29+
}
30+
31+
else {
32+
long val = Math.abs(random.nextLong());
33+
samplingScore = ((double)Math.abs(val)/ Long.MAX_VALUE);
34+
}
35+
36+
return samplingScore * 100;
37+
}
38+
39+
static int getSamplingHashCode(String input) {
40+
if (StringUtils.isNullOrEmpty(input)) {
41+
return 0;
42+
}
43+
44+
StringBuilder inputBuilder = new StringBuilder(input);
45+
while (inputBuilder.length() < 8) {
46+
inputBuilder.append(input);
47+
}
48+
49+
int hash = 5381;
50+
51+
for (int i = 0; i < inputBuilder.length(); ++i) {
52+
hash = ((hash << 5) + hash) + (int) inputBuilder.charAt(i);
53+
}
54+
55+
return hash == Integer.MIN_VALUE ? Integer.MAX_VALUE : Math.abs(hash);
56+
}
57+
}

‎core/src/main/java/com/microsoft/applicationinsights/internal/config/ParamExcludedTypeXmlElement.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package com.microsoft.applicationinsights.internal.config;
2+
3+
import javax.xml.bind.annotation.XmlElement;
4+
import javax.xml.bind.annotation.XmlRootElement;
5+
import java.util.List;
6+
7+
/**
8+
* This is the class for binding the xml array list of <ExcludedTypes>
9+
*/
10+
11+
@XmlRootElement(name = "ExcludedTypes")
12+
public class ParamExcludedTypeXmlElement {
13+
14+
public List<String> getExcludedType() {
15+
return excludedType;
16+
}
17+
18+
@XmlElement(name = "ExcludedType")
19+
public void setExcludedType(List<String> excludedType) {
20+
this.excludedType = excludedType;
21+
}
22+
23+
private List<String> excludedType;
24+
25+
}

‎core/src/main/java/com/microsoft/applicationinsights/internal/config/ParamIncludedTypeXmlElement.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
package com.microsoft.applicationinsights.internal.config;
2+
3+
import javax.xml.bind.annotation.XmlElement;
4+
import javax.xml.bind.annotation.XmlRootElement;
5+
import java.util.List;
6+
7+
/**
8+
* This class is used to bind the xml array list of <IncludeTypes>
9+
*/
10+
11+
@XmlRootElement(name = "IncludedTypes")
12+
public class ParamIncludedTypeXmlElement {
13+
14+
public List<String> getIncludedType() {
15+
return includedType;
16+
}
17+
18+
@XmlElement(name = "IncludedType")
19+
public void setIncludedType(List<String> includedType) {
20+
this.includedType = includedType;
21+
}
22+
23+
private List<String> includedType;
24+
25+
}

0 commit comments

Comments
 (0)