Skip to content

Commit 04cb0c4

Browse files
BrandonArpBrandonArp
committed
update with consuming variables
1 parent 2ad542a commit 04cb0c4

File tree

5 files changed

+153
-20
lines changed

5 files changed

+153
-20
lines changed

‎src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -133,8 +133,10 @@ public void notify(final Observable observable, final Object event) {
133133
final Matcher matcher = metricPattern.matcher(metricName);
134134
if (matcher.find()) {
135135
for (final String replacement : findAndReplace.getValue()) {
136-
final String replacedString =
136+
final RegexAndMapReplacer.Replacement rep =
137137
RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, record.getDimensions());
138+
final String replacedString = rep.getReplacement();
139+
final ImmutableList<String> consumedDimensions = rep.getVariablesMatched();
138140

139141
final int tagsStart = replacedString.indexOf(';');
140142
if (tagsStart == -1) {
@@ -143,11 +145,13 @@ public void notify(final Observable observable, final Object event) {
143145
} else {
144146
final String newMetricName = replacedString.substring(0, tagsStart);
145147
final Map<String, String> parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1));
146-
final ImmutableMap.Builder<String, String> finalTags = ImmutableMap.builder();
148+
final Map<String, String> finalTags = Maps.newHashMap();
147149
finalTags.putAll(record.getDimensions());
150+
// Remove the dimensions that we consumed in the replacement
151+
consumedDimensions.forEach(finalTags::remove);
148152
finalTags.putAll(parsedTags);
149153

150-
merge(metric.getValue(), newMetricName, mergedMetrics, finalTags.build());
154+
merge(metric.getValue(), newMetricName, mergedMetrics, ImmutableMap.copyOf(finalTags));
151155
}
152156
}
153157
//Having "found" set here means that mapping a metric to an empty list suppresses that metric

‎src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java‎

Lines changed: 38 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@
1515
*/
1616
package com.arpnetworking.utility;
1717

18+
import com.google.common.collect.ImmutableList;
1819
import com.google.common.collect.ImmutableMap;
1920

2021
import java.util.Map;
@@ -42,36 +43,38 @@ public final class RegexAndMapReplacer {
4243
* @param variables map of variables to include
4344
* @return a string with replacement tokens replaced
4445
*/
45-
public static String replaceAll(
46+
public static Replacement replaceAll(
4647
final Pattern pattern,
4748
final String input,
4849
final String replace,
4950
final ImmutableMap<String, String> variables) {
5051
final Matcher matcher = pattern.matcher(input);
5152
boolean found = matcher.find();
5253
if (found) {
54+
final ImmutableList.Builder<String> variablesUsedBuilder = ImmutableList.builder();
5355
final StringBuilder builder = new StringBuilder();
5456
int lastMatchedIndex = 0;
5557
do {
5658
builder.append(input.substring(lastMatchedIndex, matcher.start()));
5759
lastMatchedIndex = matcher.end();
58-
appendReplacement(matcher, replace, builder, variables);
60+
appendReplacement(matcher, replace, builder, variables, variablesUsedBuilder);
5961
found = matcher.find();
6062
} while (found);
6163
// Append left-over string after the matches
6264
if (lastMatchedIndex < input.length() - 1) {
6365
builder.append(input.substring(lastMatchedIndex, input.length()));
6466
}
65-
return builder.toString();
67+
return new Replacement(builder.toString(), variablesUsedBuilder.build());
6668
}
67-
return input;
69+
return new Replacement(input, ImmutableList.of());
6870
}
6971

7072
private static void appendReplacement(
7173
final Matcher matcher,
7274
final String replacement,
7375
final StringBuilder replacementBuilder,
74-
final Map<String, String> variables) {
76+
final Map<String, String> variables,
77+
final ImmutableList.Builder<String> variablesUsedBuilder) {
7578
final StringBuilder tokenBuilder = new StringBuilder();
7679
int x = -1;
7780
while (x < replacement.length() - 1) {
@@ -82,7 +85,7 @@ private static void appendReplacement(
8285
processEscapedCharacter(replacement, x, replacementBuilder);
8386
} else {
8487
if (c == '$') {
85-
x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder);
88+
x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder, variablesUsedBuilder);
8689
} else {
8790
replacementBuilder.append(c);
8891
}
@@ -110,7 +113,8 @@ private static int writeReplacementToken(
110113
final StringBuilder output,
111114
final Matcher matcher,
112115
final Map<String, String> variables,
113-
final StringBuilder tokenBuilder) {
116+
final StringBuilder tokenBuilder,
117+
final ImmutableList.Builder<String> variablesUsedBuilder) {
114118
boolean inReplaceBrackets = false;
115119
boolean tokenNumeric = true;
116120
tokenBuilder.setLength(0); // reset the shared builder
@@ -143,7 +147,7 @@ private static int writeReplacementToken(
143147
throw new IllegalArgumentException("Invalid replacement token, expected '}' at col " + x + ": " + replacement);
144148
}
145149
x++; // Consume the }
146-
output.append(getReplacement(matcher, tokenBuilder.toString(), tokenNumeric, variables));
150+
output.append(getReplacement(matcher, tokenBuilder.toString(), tokenNumeric, variables, variablesUsedBuilder));
147151
} else {
148152
// Consume until we hit a non-digit character
149153
while (x < replacement.length()) {
@@ -162,7 +166,7 @@ private static int writeReplacementToken(
162166
x,
163167
replacement));
164168
}
165-
output.append(getReplacement(matcher, tokenBuilder.toString(), true, variables));
169+
output.append(getReplacement(matcher, tokenBuilder.toString(), true, variables, variablesUsedBuilder));
166170
}
167171
return x - offset - 1;
168172
}
@@ -171,18 +175,42 @@ private static String getReplacement(
171175
final Matcher matcher,
172176
final String replaceToken,
173177
final boolean numeric,
174-
final Map<String, String> variables) {
178+
final Map<String, String> variables,
179+
final ImmutableList.Builder<String> variablesUsedBuilder) {
175180
if (numeric) {
176181
final int replaceGroup = Integer.parseInt(replaceToken);
177182
return matcher.group(replaceGroup);
178183
} else {
179184
try {
180185
return matcher.group(replaceToken);
181186
} catch (final IllegalArgumentException e) { // No group with this name
187+
variablesUsedBuilder.add(replaceToken);
182188
return variables.getOrDefault(replaceToken, "");
183189
}
184190
}
185191
}
186192

187193
private RegexAndMapReplacer() { }
194+
195+
/**
196+
* Describes the replacement string and variables used in it's creation.
197+
*/
198+
public static final class Replacement {
199+
public String getReplacement() {
200+
return _replacement;
201+
}
202+
203+
public ImmutableList<String> getVariablesMatched() {
204+
return _variablesMatched;
205+
}
206+
207+
private Replacement(final String replacement, final ImmutableList<String> variablesMatched) {
208+
209+
_replacement = replacement;
210+
_variablesMatched = variablesMatched;
211+
}
212+
213+
private final String _replacement;
214+
private final ImmutableList<String> _variablesMatched;
215+
}
188216
}

‎src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java‎

Lines changed: 102 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,8 @@
2828
import com.arpnetworking.tsdcore.model.Unit;
2929
import com.google.common.collect.ImmutableList;
3030
import com.google.common.collect.ImmutableMap;
31+
import com.google.common.collect.Lists;
32+
import com.google.common.collect.Maps;
3133
import org.junit.Assert;
3234
import org.junit.Before;
3335
import org.junit.Test;
@@ -36,6 +38,7 @@
3638

3739
import java.util.Collections;
3840
import java.util.List;
41+
import java.util.Map;
3942
import java.util.regex.Pattern;
4043

4144
/**
@@ -55,7 +58,19 @@ public void setUp() {
5558
"foo/([^/]*)/bar", ImmutableList.of("foo/bar"),
5659
"cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"),
5760
"tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"),
61+
"named/(?<animal>[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"),
5862
"tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal")))
63+
.setInject(Lists.newArrayList(
64+
new TransformingSource.DimensionInjection.Builder()
65+
.setDimension("inject")
66+
.setValue("value")
67+
.setReplaceExisting(true)
68+
.build(),
69+
new TransformingSource.DimensionInjection.Builder()
70+
.setDimension("no_over")
71+
.setValue("not_overwriting")
72+
.setReplaceExisting(false)
73+
.build()))
5974
.setSource(_mockSource);
6075
}
6176

@@ -285,8 +300,8 @@ public void testExtractTagWithCapture() {
285300
.build();
286301
assertRecordsEqual(actualRecord, expectedRecord);
287302
}
288-
@Test
289303

304+
@Test
290305
public void testInjectTagWithCapture() {
291306
final Record matchingRecord = TestBeanFactory.createRecordBuilder()
292307
.setMetrics(ImmutableMap.of(
@@ -327,6 +342,92 @@ public void testInjectTagWithCapture() {
327342
assertRecordsEqual(actualRecord, expectedRecord);
328343
}
329344

345+
@Test
346+
public void testMatchOverridesTagInCapture() {
347+
final Record matchingRecord = TestBeanFactory.createRecordBuilder()
348+
.setMetrics(ImmutableMap.of(
349+
"named/frog",
350+
TestBeanFactory.createMetricBuilder()
351+
.setType(MetricType.GAUGE)
352+
.setValues(ImmutableList.of(
353+
new Quantity.Builder()
354+
.setValue(1.23d)
355+
.setUnit(Unit.BYTE)
356+
.build()))
357+
.build()))
358+
.setDimensions(
359+
ImmutableMap.of(
360+
Key.HOST_DIMENSION_KEY, "MyHost",
361+
Key.SERVICE_DIMENSION_KEY, "MyService",
362+
Key.CLUSTER_DIMENSION_KEY, "MyCluster",
363+
"animal", "cat"))
364+
.build();
365+
366+
final Record actualRecord = mapRecord(matchingRecord);
367+
368+
final Map<String, String> expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions());
369+
expectedDimensions.put("extracted", "frog");
370+
final Record expectedRecord = TestBeanFactory.createRecordBuilder()
371+
.setAnnotations(matchingRecord.getAnnotations())
372+
.setTime(matchingRecord.getTime())
373+
.setDimensions(ImmutableMap.copyOf(expectedDimensions))
374+
.setMetrics(ImmutableMap.of(
375+
"named/extracted_animal",
376+
TestBeanFactory.createMetricBuilder()
377+
.setType(MetricType.GAUGE)
378+
.setValues(ImmutableList.of(
379+
new Quantity.Builder()
380+
.setValue(1.23d)
381+
.setUnit(Unit.BYTE)
382+
.build()))
383+
.build()))
384+
.build();
385+
assertRecordsEqual(actualRecord, expectedRecord);
386+
}
387+
388+
@Test
389+
public void testExtractOverridesExisting() {
390+
final Record matchingRecord = TestBeanFactory.createRecordBuilder()
391+
.setMetrics(ImmutableMap.of(
392+
"named/frog",
393+
TestBeanFactory.createMetricBuilder()
394+
.setType(MetricType.GAUGE)
395+
.setValues(ImmutableList.of(
396+
new Quantity.Builder()
397+
.setValue(1.23d)
398+
.setUnit(Unit.BYTE)
399+
.build()))
400+
.build()))
401+
.setDimensions(
402+
ImmutableMap.of(
403+
Key.HOST_DIMENSION_KEY, "MyHost",
404+
Key.SERVICE_DIMENSION_KEY, "MyService",
405+
Key.CLUSTER_DIMENSION_KEY, "MyCluster",
406+
"extracted", "none"))
407+
.build();
408+
409+
final Record actualRecord = mapRecord(matchingRecord);
410+
411+
final Map<String, String> expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions());
412+
expectedDimensions.put("extracted", "frog");
413+
final Record expectedRecord = TestBeanFactory.createRecordBuilder()
414+
.setAnnotations(matchingRecord.getAnnotations())
415+
.setTime(matchingRecord.getTime())
416+
.setDimensions(ImmutableMap.copyOf(expectedDimensions))
417+
.setMetrics(ImmutableMap.of(
418+
"named/extracted_animal",
419+
TestBeanFactory.createMetricBuilder()
420+
.setType(MetricType.GAUGE)
421+
.setValues(ImmutableList.of(
422+
new Quantity.Builder()
423+
.setValue(1.23d)
424+
.setUnit(Unit.BYTE)
425+
.build()))
426+
.build()))
427+
.build();
428+
assertRecordsEqual(actualRecord, expectedRecord);
429+
}
430+
330431
@Test
331432
public void testReplaceWithCaptureWithTags() {
332433
final Record matchingRecord = TestBeanFactory.createRecordBuilder()

‎src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public final class RegexAndMapBenchmarkTest {
3434
@BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000)
3535
@Test
3636
public void testRegexAndMap() {
37-
final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, ImmutableMap.of());
37+
final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, ImmutableMap.of()).getReplacement();
3838
}
3939

4040
@BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000)

‎src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -41,23 +41,23 @@ public void testInvalidEscape() {
4141
final Pattern pattern = Pattern.compile("test");
4242
final String input = "test";
4343
final String replace = "${\\avariable}"; // \a is an invalid escape sequence
44-
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of());
44+
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement();
4545
}
4646

4747
@Test(expected = IllegalArgumentException.class)
4848
public void testMissingClosingCurly() {
4949
final Pattern pattern = Pattern.compile("test");
5050
final String input = "test";
5151
final String replace = "${0"; // no ending }
52-
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of());
52+
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement();
5353
}
5454

5555
@Test(expected = IllegalArgumentException.class)
5656
public void testInvalidEscapeAtEnd() {
5757
final Pattern pattern = Pattern.compile("test");
5858
final String input = "test";
5959
final String replace = "${0}\\"; // trailing \
60-
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of());
60+
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement();
6161
}
6262

6363
@Test
@@ -74,7 +74,7 @@ public void testInvalidReplacementTokenMissingOpen() {
7474
final Pattern pattern = Pattern.compile("test");
7575
final String input = "test";
7676
final String replace = "$variable"; // replacement variable has no {
77-
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of());
77+
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement();
7878
}
7979

8080
@Test
@@ -187,7 +187,7 @@ public void testMultipleMatchPartialStaticReplace() {
187187

188188
private void testExpression(final Pattern pattern, final String input, final String replace, final String expected,
189189
final ImmutableMap<String, String> variables) {
190-
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables);
190+
final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables).getReplacement();
191191
Assert.assertEquals(expected, result);
192192
try {
193193
final String stockResult = pattern.matcher(input).replaceAll(replace);

0 commit comments

Comments
 (0)