From 2ad542adedf4cf9122533a0815d71bfd6340a9ca Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Tue, 27 Feb 2018 01:56:48 -0800 Subject: [PATCH 01/11] add transforming source to remove, inject, and transform metrics and dimensions --- jdk-wrapper.sh | 28 +- .../mad/sources/TransformingSource.java | 393 +++++++++++++++ .../utility/RegexAndMapReplacer.java | 188 +++++++ .../mad/sources/TransformingSourceTest.java | 463 ++++++++++++++++++ .../test/UnorderedRecordEquality.java | 3 +- .../utility/RegexAndMapBenchmarkTest.java | 51 ++ .../utility/RegexAndMapReplacerTest.java | 197 ++++++++ 7 files changed, 1320 insertions(+), 3 deletions(-) create mode 100644 src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java create mode 100644 src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java create mode 100644 src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java create mode 100644 src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java create mode 100644 src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java diff --git a/jdk-wrapper.sh b/jdk-wrapper.sh index c83f09f8..d8bf1596 100755 --- a/jdk-wrapper.sh +++ b/jdk-wrapper.sh @@ -40,6 +40,27 @@ safe_command() { fi } +checksum() { + l_file="$1" + checksum_exec="" + if command -v md5 > /dev/null; then + checksum_exec="md5" + elif command -v sha1sum > /dev/null; then + checksum_exec="sha1sum" + elif command -v shasum > /dev/null; then + checksum_exec="shasum" + fi + if [ -z "${checksum_exec}" ]; then + log_err "ERROR: No supported checksum command found!" + exit 1 + fi + cat "${l_file}" | ${checksum_exec} +} + +rand() { + awk 'BEGIN {srand();printf "%d\n", (rand() * 10^8);}' +} + download() { file="$1" if [ ! -f "${JDKW_PATH}/${file}" ]; then @@ -93,7 +114,10 @@ fi # Resolve latest version if [ "${JDKW_RELEASE}" = "latest" ]; then - JDKW_RELEASE=$(curl ${CURL_OPTIONS} -f -k -L -H 'Accept: application/json' "${JDKW_BASE_URI}/releases/latest" | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/') + latest_version_json="${TMPDIR:-/tmp}/jdkw-latest-version-$$.$(rand)" + safe_command "curl ${CURL_OPTIONS} -f -k -L -o \"${latest_version_json}\" -H 'Accept: application/json' \"${JDKW_BASE_URI}/releases/latest\"" + JDKW_RELEASE=$(cat "${latest_version_json}" | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/') + rm -f "${latest_version_json}" log_out "Resolved latest version to ${JDKW_RELEASE}" fi @@ -114,7 +138,7 @@ download "${JDKW_WRAPPER}" # Check whether this wrapper is the one specified for this version jdkw_download="${JDKW_PATH}/${JDKW_WRAPPER}" jdkw_current="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/$(basename "$0")" -if [ "$(cat "${jdkw_download}" | sha1sum )" != "$(cat "${jdkw_current}" | sha1sum)" ]; then +if [ "$(checksum "${jdkw_download}")" != "$(checksum "${jdkw_current}")" ]; then printf "\e[0;31m[WARNING]\e[0m Your jdk-wrapper.sh file does not match the one in your JDKW_RELEASE.\n" printf "\e[0;32mUpdate your jdk-wrapper.sh to match by running:\e[0m\n" printf "cp \"%s\" \"%s\"\n" "${jdkw_download}" "${jdkw_current}" diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java new file mode 100644 index 00000000..2b990ad9 --- /dev/null +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -0,0 +1,393 @@ +/** + * Copyright 2018 InscopeMetrics.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.metrics.mad.sources; + +import com.arpnetworking.commons.builder.OvalBuilder; +import com.arpnetworking.commons.builder.ThreadLocalBuilder; +import com.arpnetworking.commons.observer.Observable; +import com.arpnetworking.commons.observer.Observer; +import com.arpnetworking.logback.annotations.LogValue; +import com.arpnetworking.metrics.common.sources.BaseSource; +import com.arpnetworking.metrics.common.sources.Source; +import com.arpnetworking.metrics.mad.model.DefaultMetric; +import com.arpnetworking.metrics.mad.model.DefaultRecord; +import com.arpnetworking.metrics.mad.model.Metric; +import com.arpnetworking.metrics.mad.model.Record; +import com.arpnetworking.steno.LogValueMapFactory; +import com.arpnetworking.steno.Logger; +import com.arpnetworking.steno.LoggerFactory; +import com.arpnetworking.tsdcore.model.MetricType; +import com.arpnetworking.tsdcore.model.Quantity; +import com.arpnetworking.utility.RegexAndMapReplacer; +import com.google.common.base.MoreObjects; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import net.sf.oval.constraint.NotEmpty; +import net.sf.oval.constraint.NotNull; + +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Implementation of {@link Source} which wraps another {@link Source} + * and merges {@link Metric} instances within each {@link Record} + * together while, optionally, removing, injecting, and modifying dimensions + * and metrics if the name matches a regular expression with a new name generated + * through replacement of all matches in the original name. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + * @author Ville Koskela (ville dot koskela at inscopemetrics dot com) + */ +public final class TransformingSource extends BaseSource { + + @Override + public void start() { + _source.start(); + } + + @Override + public void stop() { + _source.stop(); + } + + /** + * Generate a Steno log compatible representation. + * + * @return Steno log compatible representation. + */ + @LogValue + public Object toLogValue() { + return LogValueMapFactory.builder(this) + .put("source", _source) + .put("findAndReplace", _findAndReplace) + .build(); + } + + @Override + public String toString() { + return toLogValue().toString(); + } + + private TransformingSource(final Builder builder) { + super(builder); + _source = builder._source; + + _findAndReplace = Maps.newHashMapWithExpectedSize(builder._findAndReplace.size()); + for (final Map.Entry> entry : builder._findAndReplace.entrySet()) { + _findAndReplace.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); + } + + _source.attach(new TransformingObserver(this, _findAndReplace)); + } + + private final Source _source; + private final Map> _findAndReplace; + + private static final Logger LOGGER = LoggerFactory.getLogger(TransformingSource.class); + private static final Splitter.MapSplitter TAG_SPLITTER = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('='); + + // NOTE: Package private for testing + /* package private */ static final class TransformingObserver implements Observer { + + /* package private */ TransformingObserver(final TransformingSource source, final Map> findAndReplace) { + _source = source; + _findAndReplace = findAndReplace; + } + + @Override + public void notify(final Observable observable, final Object event) { + if (!(event instanceof Record)) { + LOGGER.error() + .setMessage("Observed unsupported event") + .addData("event", event) + .log(); + return; + } + + // Merge the metrics in the record together + final Record record = (Record) event; + final Map, Map> mergedMetrics = Maps.newHashMap(); + for (final Map.Entry metric : record.getMetrics().entrySet()) { + boolean found = false; + final String metricName = metric.getKey(); + for (final Map.Entry> findAndReplace : _findAndReplace.entrySet()) { + final Pattern metricPattern = findAndReplace.getKey(); + final Matcher matcher = metricPattern.matcher(metricName); + if (matcher.find()) { + for (final String replacement : findAndReplace.getValue()) { + final String replacedString = + RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, record.getDimensions()); + + final int tagsStart = replacedString.indexOf(';'); + if (tagsStart == -1) { + // We just have a metric name. Optimize for this common case + merge(metric.getValue(), replacedString, mergedMetrics, record.getDimensions()); + } else { + final String newMetricName = replacedString.substring(0, tagsStart); + final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); + final ImmutableMap.Builder finalTags = ImmutableMap.builder(); + finalTags.putAll(record.getDimensions()); + finalTags.putAll(parsedTags); + + merge(metric.getValue(), newMetricName, mergedMetrics, finalTags.build()); + } + } + //Having "found" set here means that mapping a metric to an empty list suppresses that metric + found = true; + } + } + if (!found) { + merge(metric.getValue(), metricName, mergedMetrics, record.getDimensions()); + } + } + + // Raise the merged record event with this source's observers + // NOTE: Do not leak instances of MergingMetric since it is mutable + for (Map.Entry, Map> entry : mergedMetrics.entrySet()) { + _source.notify( + ThreadLocalBuilder.build( + DefaultRecord.Builder.class, + b1 -> b1.setMetrics( + entry.getValue().entrySet().stream().collect( + ImmutableMap.toImmutableMap( + Map.Entry::getKey, + e -> ThreadLocalBuilder.clone( + e.getValue(), + DefaultMetric.Builder.class)))) + .setId(record.getId()) + .setTime(record.getTime()) + .setAnnotations(record.getAnnotations()) + .setDimensions(entry.getKey()))); + } + } + + private void merge(final Metric metric, final String key, + final Map, Map> mergedMetrics, + final ImmutableMap dimensions) { + + final Map mergedMetricsForDimensions = mergedMetrics.computeIfAbsent(dimensions, k -> Maps.newHashMap()); + final MergingMetric mergedMetric = mergedMetricsForDimensions.get(key); + if (mergedMetric == null) { + // This is the first time this metric is being merged into + mergedMetricsForDimensions.put(key, new MergingMetric(metric)); + } else if (!mergedMetric.isMergable(metric)) { + // This instance of the metric is not mergable with previous + LOGGER.error() + .setMessage("Discarding metric") + .addData("reason", "failed to merge") + .addData("metric", metric) + .addData("mergedMetric", mergedMetric) + .log(); + } else { + // Merge the new instance in + mergedMetric.merge(metric); + } + } + + private final TransformingSource _source; + private final Map> _findAndReplace; + } + + // NOTE: Package private for testing + /* package private */ static final class MergingMetric implements Metric { + + /* package private */ MergingMetric(final Metric metric) { + _type = metric.getType(); + _values.addAll(metric.getValues()); + } + + public boolean isMergable(final Metric metric) { + return _type.equals(metric.getType()); + } + + public void merge(final Metric metric) { + if (!isMergable(metric)) { + throw new IllegalArgumentException(String.format("Metric cannot be merged; metric=%s", metric)); + } + _values.addAll(metric.getValues()); + } + + @Override + public MetricType getType() { + return _type; + } + + @Override + public ImmutableList getValues() { + return _values.build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", Integer.toHexString(System.identityHashCode(this))) + .add("Type", _type) + .add("Values", _values) + .toString(); + } + + private final MetricType _type; + private final ImmutableList.Builder _values = ImmutableList.builder(); + } + + /** + * Represents a dimension to inject and whether or not it should overwrite the existing value (if any). + */ + public static final class DimensionInjection { + public String getDimension() { + return _dimension; + } + + public String getValue() { + return _value; + } + + public boolean isReplaceExisting() { + return _replaceExisting; + } + + private DimensionInjection(final Builder builder) { + _dimension = builder._dimension; + _value = builder._value; + _replaceExisting = builder._replaceExisting; + } + + private final String _dimension; + private final String _value; + private final boolean _replaceExisting; + + /** + * Implementation of the Builder pattern for {@link DimensionInjection}. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ + public static final class Builder extends OvalBuilder { + /** + * Public constructor. + */ + public Builder() { + super(DimensionInjection::new); + } + + /** + * Sets the dimension. Required. Cannot be null. Cannot be empty. + * + * @param value The dimension to inject. + * @return This instance of {@link Builder}. + */ + public Builder setDimension(final String value) { + _dimension = value; + return this; + } + + /** + * Sets the value. Required. Cannot be null. Cannot be empty. + * + * @param value The value to inject. + * @return This instance of {@link Builder}. + */ + public Builder setValue(final String value) { + _value = value; + return this; + } + + /** + * Whether to override existing dimension of this name. Optional. Cannot be null. Defaults to true. + * + * @param value true to replace existing dimension value + * @return This instance of {@link Builder}. + */ + public Builder setReplaceExisting(final Boolean value) { + _replaceExisting = value; + return this; + } + + @NotNull + @NotEmpty + private String _dimension; + @NotNull + @NotEmpty + private String _value; + @NotNull + private Boolean _replaceExisting = true; + } + } + + /** + * Implementation of builder pattern for {@link TransformingSource}. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ + public static final class Builder extends BaseSource.Builder { + + /** + * Public constructor. + */ + public Builder() { + super(TransformingSource::new); + } + + /** + * Sets the underlying source. Cannot be null. + * + * @param value The underlying source. + * @return This instance of Builder. + */ + public Builder setSource(final Source value) { + _source = value; + return this; + } + + /** + * Sets find and replace expression map. Optional. Cannot be null. Defaults to empty. + * + * @param value The find and replace expression map. + * @return This instance of Builder. + */ + public Builder setFindAndReplace(final Map> value) { + _findAndReplace = value; + return this; + } + + /** + * Sets dimensions to inject. Optional. Cannot be null. Defaults to empty. + * + * @param value List of dimensions to inject. + * @return This instance of Builder. + */ + public Builder setInject(final List value) { + _inject = value; + return this; + } + + @Override + protected Builder self() { + return this; + } + + @NotNull + private Source _source; + @NotNull + private Map> _findAndReplace = Maps.newHashMap(); + @NotNull + private List _inject = Lists.newArrayList(); + } +} diff --git a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java new file mode 100644 index 00000000..a5ce35a9 --- /dev/null +++ b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java @@ -0,0 +1,188 @@ +/** + * Copyright 2018 Inscope Metrics, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.utility; + +import com.google.common.collect.ImmutableMap; + +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * A regex replacement utility that can also replace tokens not found in the regex. + * + * $n where n is a number in the replace string is replaced by the pattern's match group n + * ${name} in the replace string is replaced by the pattern's named capture, or by the value of the variable + * with that name from the variables map + * \ is used as an escape character and may be used to escape '$', '{', and '}' characters so that they will + * be treated as literals + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ +public final class RegexAndMapReplacer { + /** + * Replaces all instances of $n (where n is 0-9) with regex match groups, ${var} with regex capture group or variable (from map) 'var'. + * + * @param pattern pattern to use + * @param input input string to match against + * @param replace replacement string + * @param variables map of variables to include + * @return a string with replacement tokens replaced + */ + public static String replaceAll( + final Pattern pattern, + final String input, + final String replace, + final ImmutableMap variables) { + final Matcher matcher = pattern.matcher(input); + boolean found = matcher.find(); + if (found) { + final StringBuilder builder = new StringBuilder(); + int lastMatchedIndex = 0; + do { + builder.append(input.substring(lastMatchedIndex, matcher.start())); + lastMatchedIndex = matcher.end(); + appendReplacement(matcher, replace, builder, variables); + found = matcher.find(); + } while (found); + // Append left-over string after the matches + if (lastMatchedIndex < input.length() - 1) { + builder.append(input.substring(lastMatchedIndex, input.length())); + } + return builder.toString(); + } + return input; + } + + private static void appendReplacement( + final Matcher matcher, + final String replacement, + final StringBuilder replacementBuilder, + final Map variables) { + final StringBuilder tokenBuilder = new StringBuilder(); + int x = -1; + while (x < replacement.length() - 1) { + x++; + final char c = replacement.charAt(x); + if (c == '\\') { + x++; + processEscapedCharacter(replacement, x, replacementBuilder); + } else { + if (c == '$') { + x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder); + } else { + replacementBuilder.append(c); + } + } + } + } + + private static void processEscapedCharacter(final String replacement, final int x, final StringBuilder builder) { + if (x >= replacement.length()) { + throw new IllegalArgumentException( + String.format("Improper escaping in replacement, must not have trailing '\\' at col %d: %s", x, replacement)); + } + final Character c = replacement.charAt(x); + if (c == '\\' || c == '$' || c == '{' || c == '}') { + builder.append(c); + } else { + throw new IllegalArgumentException( + String.format("Improperly escaped '%s' in replacement at col %d: %s", c, x, replacement)); + } + } + + private static int writeReplacementToken( + final String replacement, + final int offset, + final StringBuilder output, + final Matcher matcher, + final Map variables, + final StringBuilder tokenBuilder) { + boolean inReplaceBrackets = false; + boolean tokenNumeric = true; + tokenBuilder.setLength(0); // reset the shared builder + int x = offset + 1; + char c = replacement.charAt(x); + + // Optionally consume the opening brace + if (c == '{') { + inReplaceBrackets = true; + x++; + c = replacement.charAt(x); + } + + if (inReplaceBrackets) { + // Consume until we hit the } + while (x < replacement.length() - 1 && c != '}') { + if (c == '\\') { + x++; + processEscapedCharacter(replacement, x, tokenBuilder); + } else { + tokenBuilder.append(c); + } + if (tokenNumeric && !Character.isDigit(c)) { + tokenNumeric = false; + } + x++; + c = replacement.charAt(x); + } + if (c != '}') { + throw new IllegalArgumentException("Invalid replacement token, expected '}' at col " + x + ": " + replacement); + } + x++; // Consume the } + output.append(getReplacement(matcher, tokenBuilder.toString(), tokenNumeric, variables)); + } else { + // Consume until we hit a non-digit character + while (x < replacement.length()) { + c = replacement.charAt(x); + if (Character.isDigit(c)) { + tokenBuilder.append(c); + } else { + break; + } + x++; + } + if (tokenBuilder.length() == 0) { + throw new IllegalArgumentException( + String.format( + "Invalid replacement token, non-numeric tokens must be surrounded by { } at col %d: %s", + x, + replacement)); + } + output.append(getReplacement(matcher, tokenBuilder.toString(), true, variables)); + } + return x - offset - 1; + } + + private static String getReplacement( + final Matcher matcher, + final String replaceToken, + final boolean numeric, + final Map variables) { + if (numeric) { + final int replaceGroup = Integer.parseInt(replaceToken); + return matcher.group(replaceGroup); + } else { + try { + return matcher.group(replaceToken); + } catch (final IllegalArgumentException e) { // No group with this name + return variables.getOrDefault(replaceToken, ""); + } + } + } + + private RegexAndMapReplacer() { } +} diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java new file mode 100644 index 00000000..4ababe3e --- /dev/null +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -0,0 +1,463 @@ +/** + * Copyright 2018 InscopeMetrics.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.metrics.mad.sources; + +import com.arpnetworking.commons.observer.Observable; +import com.arpnetworking.commons.observer.Observer; +import com.arpnetworking.metrics.common.sources.Source; +import com.arpnetworking.metrics.mad.model.Record; +import com.arpnetworking.metrics.mad.sources.TransformingSource.MergingMetric; +import com.arpnetworking.test.TestBeanFactory; +import com.arpnetworking.test.UnorderedRecordEquality; +import com.arpnetworking.tsdcore.model.Key; +import com.arpnetworking.tsdcore.model.MetricType; +import com.arpnetworking.tsdcore.model.Quantity; +import com.arpnetworking.tsdcore.model.Unit; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.List; +import java.util.regex.Pattern; + +/** + * Tests for the MergingSource class. + * + * @author Ville Koskela (ville dot koskela at inscopemetrics dot com) + */ +public class TransformingSourceTest { + + @Before + public void setUp() { + _mockObserver = Mockito.mock(Observer.class); + _mockSource = Mockito.mock(Source.class); + _transformingSourceBuilder = new TransformingSource.Builder() + .setName("TransformingSourceTest") + .setFindAndReplace(ImmutableMap.of( + "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), + "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), + "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), + "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))) + .setSource(_mockSource); + } + + @Test + public void testAttach() { + _transformingSourceBuilder.build(); + Mockito.verify(_mockSource).attach(Mockito.any(Observer.class)); + } + + @Test + public void testStart() { + _transformingSourceBuilder.build().start(); + Mockito.verify(_mockSource).start(); + } + + @Test + public void testStop() { + _transformingSourceBuilder.build().stop(); + Mockito.verify(_mockSource).stop(); + } + + @Test + public void testToString() { + final String asString = _transformingSourceBuilder.build().toString(); + Assert.assertNotNull(asString); + Assert.assertFalse(asString.isEmpty()); + } + + @Test + public void testMergingObserverInvalidEvent() { + final TransformingSource transformingSource = new TransformingSource.Builder() + .setName("testMergingObserverInvalidEventTransformingSource") + .setSource(_mockSource) + .setFindAndReplace(Collections.>emptyMap()) + .build(); + Mockito.reset(_mockSource); + new TransformingSource.TransformingObserver( + transformingSource, + Collections.>emptyMap()) + .notify(OBSERVABLE, "Not a Record"); + Mockito.verifyZeroInteractions(_mockSource); + } + + @Test(expected = IllegalArgumentException.class) + public void testMergingMetricMergeMismatchedTypes() { + final MergingMetric mergingMetric = new MergingMetric( + TestBeanFactory.createMetricBuilder() + .setType(MetricType.COUNTER) + .build()); + mergingMetric.merge(TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .build()); + } + + @Test + public void testMergeNotMatch() { + final Record nonMatchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "does_not_match", + TestBeanFactory.createMetric())) + .build(); + + final Record actualRecord = mapRecord(nonMatchingRecord); + + assertRecordsEqual(actualRecord, nonMatchingRecord); + } + + @Test + public void testMergeTwoGauges() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "foo/1/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build(), + "foo/2/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "foo/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build(), + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testDropMetricOfDifferentType() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "foo/1/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(1.23d).build())) + .build(), + "foo/2/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.TIMER) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(2.46d).build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord1 = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "foo/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(1.23d).build())) + .build())) + .build(); + final Record expectedRecord2 = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "foo/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.TIMER) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(2.46d).build())) + .build())) + .build(); + + Assert.assertTrue( + String.format("expected1=%s OR expected2=%s, actual=%s", expectedRecord1, expectedRecord2, actualRecord), + UnorderedRecordEquality.equals(expectedRecord1, actualRecord) + || UnorderedRecordEquality.equals(expectedRecord2, actualRecord)); + } + + @Test + public void testReplaceWithCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "cat/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "cat/dog/sheep", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testExtractTagWithCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "tagged/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.builder().putAll(matchingRecord.getDimensions()).put("animal", "sheep").build()) + .setMetrics(ImmutableMap.of( + "tagged/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + @Test + + public void testInjectTagWithCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "tagged/foo/animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "animal", "frog")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(matchingRecord.getDimensions()) + .setMetrics(ImmutableMap.of( + "tagged/frog/animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testReplaceWithCaptureWithTags() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "cat/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "cat/dog/sheep", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testMultipleMatches() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "cat/bear/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build(), + "cat/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "cat/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build(), + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build(), + "cat/dog/sheep", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build(), + "cat/dog/bear", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + private void assertRecordsEqual(final Record actualRecord, final Record expectedRecord) { + Assert.assertTrue( + String.format("expected=%s, actual=%s", expectedRecord, actualRecord), + UnorderedRecordEquality.equals(expectedRecord, actualRecord)); + } + + private Record mapRecord(final Record record) { + final Source transformingSource = _transformingSourceBuilder.build(); + transformingSource.attach(_mockObserver); + notify(_mockSource, record); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(Record.class); + Mockito.verify(_mockObserver).notify(Mockito.same(transformingSource), argument.capture()); + return argument.getValue(); + } + + private static void notify(final Observable observable, final Object event) { + final ArgumentCaptor argument = ArgumentCaptor.forClass(Observer.class); + Mockito.verify(observable).attach(argument.capture()); + for (final Observer observer : argument.getAllValues()) { + observer.notify(observable, event); + } + } + + private Observer _mockObserver; + private Source _mockSource; + private TransformingSource.Builder _transformingSourceBuilder; + + private static final Observable OBSERVABLE = new Observable() { + @Override + public void attach(final Observer observer) { + } + + @Override + public void detach(final Observer observer) { + } + }; +} diff --git a/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java b/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java index 04d1e446..d5906965 100644 --- a/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java +++ b/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java @@ -42,7 +42,8 @@ public final class UnorderedRecordEquality { */ public static boolean equals(final Record r1, final Record r2) { if (!r1.getTime().equals(r2.getTime()) - || !r1.getAnnotations().equals(r2.getAnnotations())) { + || !r1.getAnnotations().equals(r2.getAnnotations()) + || !r1.getDimensions().equals(r2.getDimensions())) { return false; } diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java new file mode 100644 index 00000000..1cfe2300 --- /dev/null +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java @@ -0,0 +1,51 @@ +/** + * Copyright 2018 Inscope Metrics, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.utility; + +import com.carrotsearch.junitbenchmarks.BenchmarkOptions; +import com.carrotsearch.junitbenchmarks.BenchmarkRule; +import com.google.common.collect.ImmutableMap; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import java.util.regex.Pattern; + +/** + * Benchmark tests comparing the RegexAndMapReplacer class against the built-in regex replacement. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ +public final class RegexAndMapBenchmarkTest { + + @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) + @Test + public void testRegexAndMap() { + final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, ImmutableMap.of()); + } + + @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) + @Test + public void testRegex() { + final String result = PATTERN.matcher(INPUT).replaceAll(REPLACE); + } + + @Rule + public TestRule _benchmarkRun = new BenchmarkRule(); + private static final String REPLACE = "this is a ${g1} pattern called ${g2}"; + private static final Pattern PATTERN = Pattern.compile("(?test)/pattern/(?foo)"); + private static final String INPUT = "test/pattern/foo"; +} diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java new file mode 100644 index 00000000..fc1eab39 --- /dev/null +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java @@ -0,0 +1,197 @@ +/** + * Copyright 2018 Inscope Metrics, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.utility; + +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.Test; + +import java.util.regex.Pattern; + +/** + * Tests for the {@link RegexAndMapReplacer} class. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ +public final class RegexAndMapReplacerTest { + @Test + public void testNoMatch() { + final Pattern pattern = Pattern.compile("test"); + final String input = "wont match"; + final String replace = "$0"; + final String expected = "wont match"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidEscape() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${\\avariable}"; // \a is an invalid escape sequence + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + } + + @Test(expected = IllegalArgumentException.class) + public void testMissingClosingCurly() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${0"; // no ending } + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidEscapeAtEnd() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${0}\\"; // trailing \ + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + } + + @Test + public void testNumericWithClosingCurly() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "$0}"; + final String expected = "test}"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidReplacementTokenMissingOpen() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "$variable"; // replacement variable has no { + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + } + + @Test + public void testGroup0Replace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "$0"; + final String expected = "test"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchFullStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "replace"; + final String expected = "replace"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test string"; + final String replace = "replace"; + final String expected = "replace string"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialStaticReplacePrefix() { + final Pattern pattern = Pattern.compile("test"); + final String input = "some test string"; + final String replace = "replace"; + final String expected = "some replace string"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialMultipleGroupNumberReplace() { + final Pattern pattern = Pattern.compile("(test)/pattern/(foo)"); + final String input = "test/pattern/foo"; + final String replace = "this is a $1 pattern called $2"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialMultipleGroupNameReplace() { + final Pattern pattern = Pattern.compile("(?test)/pattern/(?foo)"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialMultipleVariableReplace() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "test", "g2", "foo")); + } + + @Test + public void testSingleMatchPartialMultipleVariableWithEscapeReplace() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern \\\\called\\\\ ${g2}"; + final String expected = "this is a test pattern \\called\\ foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "test", "g2", "foo")); + } + + @Test + public void testSingleMatchPartialMultipleVariableWithEscapeTokenReplace() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${\\\\g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("\\g1", "test", "g2", "foo")); + } + + @Test + public void testSingleMatchPartialMultipleGroupNameOverridesVariablesReplace() { + final Pattern pattern = Pattern.compile("(?test)/pattern/(?foo)"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "bad", "g2", "value")); + } + + @Test + public void testMultipleMatchFullStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "testtest"; + final String replace = "replace"; + final String expected = "replacereplace"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testMultipleMatchPartialStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test string test"; + final String replace = "replace"; + final String expected = "replace string replace"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + private void testExpression(final Pattern pattern, final String input, final String replace, final String expected, + final ImmutableMap variables) { + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables); + Assert.assertEquals(expected, result); + try { + final String stockResult = pattern.matcher(input).replaceAll(replace); + Assert.assertEquals(expected, stockResult); + } catch (final IllegalArgumentException ignored) { } + } +} From 04cb0c47989cf58607f7b077a7cd59069f69a665 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Sun, 1 Apr 2018 23:51:17 -0700 Subject: [PATCH 02/11] update with consuming variables --- .../mad/sources/TransformingSource.java | 10 +- .../utility/RegexAndMapReplacer.java | 48 ++++++-- .../mad/sources/TransformingSourceTest.java | 103 +++++++++++++++++- .../utility/RegexAndMapBenchmarkTest.java | 2 +- .../utility/RegexAndMapReplacerTest.java | 10 +- 5 files changed, 153 insertions(+), 20 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index 2b990ad9..2c933ca2 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -133,8 +133,10 @@ public void notify(final Observable observable, final Object event) { final Matcher matcher = metricPattern.matcher(metricName); if (matcher.find()) { for (final String replacement : findAndReplace.getValue()) { - final String replacedString = + final RegexAndMapReplacer.Replacement rep = RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, record.getDimensions()); + final String replacedString = rep.getReplacement(); + final ImmutableList consumedDimensions = rep.getVariablesMatched(); final int tagsStart = replacedString.indexOf(';'); if (tagsStart == -1) { @@ -143,11 +145,13 @@ public void notify(final Observable observable, final Object event) { } else { final String newMetricName = replacedString.substring(0, tagsStart); final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); - final ImmutableMap.Builder finalTags = ImmutableMap.builder(); + final Map finalTags = Maps.newHashMap(); finalTags.putAll(record.getDimensions()); + // Remove the dimensions that we consumed in the replacement + consumedDimensions.forEach(finalTags::remove); finalTags.putAll(parsedTags); - merge(metric.getValue(), newMetricName, mergedMetrics, finalTags.build()); + merge(metric.getValue(), newMetricName, mergedMetrics, ImmutableMap.copyOf(finalTags)); } } //Having "found" set here means that mapping a metric to an empty list suppresses that metric diff --git a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java index a5ce35a9..bbc72487 100644 --- a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java +++ b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java @@ -15,6 +15,7 @@ */ package com.arpnetworking.utility; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import java.util.Map; @@ -42,7 +43,7 @@ public final class RegexAndMapReplacer { * @param variables map of variables to include * @return a string with replacement tokens replaced */ - public static String replaceAll( + public static Replacement replaceAll( final Pattern pattern, final String input, final String replace, @@ -50,28 +51,30 @@ public static String replaceAll( final Matcher matcher = pattern.matcher(input); boolean found = matcher.find(); if (found) { + final ImmutableList.Builder variablesUsedBuilder = ImmutableList.builder(); final StringBuilder builder = new StringBuilder(); int lastMatchedIndex = 0; do { builder.append(input.substring(lastMatchedIndex, matcher.start())); lastMatchedIndex = matcher.end(); - appendReplacement(matcher, replace, builder, variables); + appendReplacement(matcher, replace, builder, variables, variablesUsedBuilder); found = matcher.find(); } while (found); // Append left-over string after the matches if (lastMatchedIndex < input.length() - 1) { builder.append(input.substring(lastMatchedIndex, input.length())); } - return builder.toString(); + return new Replacement(builder.toString(), variablesUsedBuilder.build()); } - return input; + return new Replacement(input, ImmutableList.of()); } private static void appendReplacement( final Matcher matcher, final String replacement, final StringBuilder replacementBuilder, - final Map variables) { + final Map variables, + final ImmutableList.Builder variablesUsedBuilder) { final StringBuilder tokenBuilder = new StringBuilder(); int x = -1; while (x < replacement.length() - 1) { @@ -82,7 +85,7 @@ private static void appendReplacement( processEscapedCharacter(replacement, x, replacementBuilder); } else { if (c == '$') { - x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder); + x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder, variablesUsedBuilder); } else { replacementBuilder.append(c); } @@ -110,7 +113,8 @@ private static int writeReplacementToken( final StringBuilder output, final Matcher matcher, final Map variables, - final StringBuilder tokenBuilder) { + final StringBuilder tokenBuilder, + final ImmutableList.Builder variablesUsedBuilder) { boolean inReplaceBrackets = false; boolean tokenNumeric = true; tokenBuilder.setLength(0); // reset the shared builder @@ -143,7 +147,7 @@ private static int writeReplacementToken( throw new IllegalArgumentException("Invalid replacement token, expected '}' at col " + x + ": " + replacement); } x++; // Consume the } - output.append(getReplacement(matcher, tokenBuilder.toString(), tokenNumeric, variables)); + output.append(getReplacement(matcher, tokenBuilder.toString(), tokenNumeric, variables, variablesUsedBuilder)); } else { // Consume until we hit a non-digit character while (x < replacement.length()) { @@ -162,7 +166,7 @@ private static int writeReplacementToken( x, replacement)); } - output.append(getReplacement(matcher, tokenBuilder.toString(), true, variables)); + output.append(getReplacement(matcher, tokenBuilder.toString(), true, variables, variablesUsedBuilder)); } return x - offset - 1; } @@ -171,7 +175,8 @@ private static String getReplacement( final Matcher matcher, final String replaceToken, final boolean numeric, - final Map variables) { + final Map variables, + final ImmutableList.Builder variablesUsedBuilder) { if (numeric) { final int replaceGroup = Integer.parseInt(replaceToken); return matcher.group(replaceGroup); @@ -179,10 +184,33 @@ private static String getReplacement( try { return matcher.group(replaceToken); } catch (final IllegalArgumentException e) { // No group with this name + variablesUsedBuilder.add(replaceToken); return variables.getOrDefault(replaceToken, ""); } } } private RegexAndMapReplacer() { } + + /** + * Describes the replacement string and variables used in it's creation. + */ + public static final class Replacement { + public String getReplacement() { + return _replacement; + } + + public ImmutableList getVariablesMatched() { + return _variablesMatched; + } + + private Replacement(final String replacement, final ImmutableList variablesMatched) { + + _replacement = replacement; + _variablesMatched = variablesMatched; + } + + private final String _replacement; + private final ImmutableList _variablesMatched; + } } diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index 4ababe3e..c0810bb6 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -28,6 +28,8 @@ import com.arpnetworking.tsdcore.model.Unit; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Before; import org.junit.Test; @@ -36,6 +38,7 @@ import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.regex.Pattern; /** @@ -55,7 +58,19 @@ public void setUp() { "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), + "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))) + .setInject(Lists.newArrayList( + new TransformingSource.DimensionInjection.Builder() + .setDimension("inject") + .setValue("value") + .setReplaceExisting(true) + .build(), + new TransformingSource.DimensionInjection.Builder() + .setDimension("no_over") + .setValue("not_overwriting") + .setReplaceExisting(false) + .build())) .setSource(_mockSource); } @@ -285,8 +300,8 @@ public void testExtractTagWithCapture() { .build(); assertRecordsEqual(actualRecord, expectedRecord); } - @Test + @Test public void testInjectTagWithCapture() { final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( @@ -327,6 +342,92 @@ public void testInjectTagWithCapture() { assertRecordsEqual(actualRecord, expectedRecord); } + @Test + public void testMatchOverridesTagInCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "named/frog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "animal", "cat")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("extracted", "frog"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "named/extracted_animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testExtractOverridesExisting() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "named/frog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "extracted", "none")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("extracted", "frog"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "named/extracted_animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + @Test public void testReplaceWithCaptureWithTags() { final Record matchingRecord = TestBeanFactory.createRecordBuilder() diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java index 1cfe2300..b9b18dc9 100644 --- a/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java @@ -34,7 +34,7 @@ public final class RegexAndMapBenchmarkTest { @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) @Test public void testRegexAndMap() { - final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, ImmutableMap.of()); + final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, ImmutableMap.of()).getReplacement(); } @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java index fc1eab39..4c56cd8b 100644 --- a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java @@ -41,7 +41,7 @@ public void testInvalidEscape() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "${\\avariable}"; // \a is an invalid escape sequence - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); } @Test(expected = IllegalArgumentException.class) @@ -49,7 +49,7 @@ public void testMissingClosingCurly() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "${0"; // no ending } - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); } @Test(expected = IllegalArgumentException.class) @@ -57,7 +57,7 @@ public void testInvalidEscapeAtEnd() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "${0}\\"; // trailing \ - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); } @Test @@ -74,7 +74,7 @@ public void testInvalidReplacementTokenMissingOpen() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "$variable"; // replacement variable has no { - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); } @Test @@ -187,7 +187,7 @@ public void testMultipleMatchPartialStaticReplace() { private void testExpression(final Pattern pattern, final String input, final String replace, final String expected, final ImmutableMap variables) { - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables).getReplacement(); Assert.assertEquals(expected, result); try { final String stockResult = pattern.matcher(input).replaceAll(replace); From d53e491ef35c5e43cff6a86df9b1330675ba44a1 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Tue, 3 Apr 2018 00:19:55 -0700 Subject: [PATCH 03/11] add and remove dimensions --- .../mad/sources/TransformingSource.java | 110 ++++++---- .../mad/sources/TransformingSourceTest.java | 194 ++++++++++++++++-- 2 files changed, 248 insertions(+), 56 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index 2c933ca2..ae379dcc 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -36,11 +36,11 @@ import com.google.common.base.Splitter; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import net.sf.oval.constraint.NotEmpty; import net.sf.oval.constraint.NotNull; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.regex.Matcher; @@ -90,16 +90,23 @@ private TransformingSource(final Builder builder) { super(builder); _source = builder._source; - _findAndReplace = Maps.newHashMapWithExpectedSize(builder._findAndReplace.size()); + final ImmutableMap.Builder> findReplaceBuilder = + ImmutableMap.builderWithExpectedSize(builder._findAndReplace.size()); for (final Map.Entry> entry : builder._findAndReplace.entrySet()) { - _findAndReplace.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); + findReplaceBuilder.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); } + _findAndReplace = findReplaceBuilder.build(); - _source.attach(new TransformingObserver(this, _findAndReplace)); + _inject = builder._inject; + _remove = builder._remove; + + _source.attach(new TransformingObserver(this, _findAndReplace, _inject, _remove)); } private final Source _source; - private final Map> _findAndReplace; + private final ImmutableMap> _findAndReplace; + private final ImmutableMap _inject; + private final ImmutableList _remove; private static final Logger LOGGER = LoggerFactory.getLogger(TransformingSource.class); private static final Splitter.MapSplitter TAG_SPLITTER = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('='); @@ -107,9 +114,15 @@ private TransformingSource(final Builder builder) { // NOTE: Package private for testing /* package private */ static final class TransformingObserver implements Observer { - /* package private */ TransformingObserver(final TransformingSource source, final Map> findAndReplace) { + /* package private */ TransformingObserver( + final TransformingSource source, + final Map> findAndReplace, + final ImmutableMap inject, + final ImmutableList remove) { _source = source; _findAndReplace = findAndReplace; + _inject = inject; + _remove = remove; } @Override @@ -141,17 +154,19 @@ public void notify(final Observable observable, final Object event) { final int tagsStart = replacedString.indexOf(';'); if (tagsStart == -1) { // We just have a metric name. Optimize for this common case - merge(metric.getValue(), replacedString, mergedMetrics, record.getDimensions()); + merge( + metric.getValue(), + replacedString, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), ImmutableList.of())); } else { final String newMetricName = replacedString.substring(0, tagsStart); final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); - final Map finalTags = Maps.newHashMap(); - finalTags.putAll(record.getDimensions()); - // Remove the dimensions that we consumed in the replacement - consumedDimensions.forEach(finalTags::remove); - finalTags.putAll(parsedTags); - - merge(metric.getValue(), newMetricName, mergedMetrics, ImmutableMap.copyOf(finalTags)); + merge( + metric.getValue(), + newMetricName, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), parsedTags, consumedDimensions)); } } //Having "found" set here means that mapping a metric to an empty list suppresses that metric @@ -159,7 +174,11 @@ public void notify(final Observable observable, final Object event) { } } if (!found) { - merge(metric.getValue(), metricName, mergedMetrics, record.getDimensions()); + merge( + metric.getValue(), + metricName, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), ImmutableList.of())); } } @@ -183,6 +202,24 @@ public void notify(final Observable observable, final Object event) { } } + private ImmutableMap getModifiedDimensions( + final ImmutableMap inputDimensions, + final Map add, + final ImmutableList remove) { + final Map finalTags = Maps.newHashMap(); + finalTags.putAll(inputDimensions); + // Remove the dimensions that we consumed in the replacement + remove.forEach(finalTags::remove); + _remove.forEach(finalTags::remove); + _inject.forEach( + (key, inject) -> + finalTags.compute(key, (k, oldValue) -> + inject.isReplaceExisting() || oldValue == null ? inject.getValue() : oldValue)); + finalTags.putAll(add); + + return ImmutableMap.copyOf(finalTags); + } + private void merge(final Metric metric, final String key, final Map, Map> mergedMetrics, final ImmutableMap dimensions) { @@ -208,6 +245,8 @@ private void merge(final Metric metric, final String key, private final TransformingSource _source; private final Map> _findAndReplace; + private final ImmutableMap _inject; + private final ImmutableList _remove; } // NOTE: Package private for testing @@ -256,10 +295,6 @@ public String toString() { * Represents a dimension to inject and whether or not it should overwrite the existing value (if any). */ public static final class DimensionInjection { - public String getDimension() { - return _dimension; - } - public String getValue() { return _value; } @@ -269,12 +304,10 @@ public boolean isReplaceExisting() { } private DimensionInjection(final Builder builder) { - _dimension = builder._dimension; _value = builder._value; _replaceExisting = builder._replaceExisting; } - private final String _dimension; private final String _value; private final boolean _replaceExisting; @@ -291,17 +324,6 @@ public Builder() { super(DimensionInjection::new); } - /** - * Sets the dimension. Required. Cannot be null. Cannot be empty. - * - * @param value The dimension to inject. - * @return This instance of {@link Builder}. - */ - public Builder setDimension(final String value) { - _dimension = value; - return this; - } - /** * Sets the value. Required. Cannot be null. Cannot be empty. * @@ -324,9 +346,6 @@ public Builder setReplaceExisting(final Boolean value) { return this; } - @NotNull - @NotEmpty - private String _dimension; @NotNull @NotEmpty private String _value; @@ -366,7 +385,7 @@ public Builder setSource(final Source value) { * @param value The find and replace expression map. * @return This instance of Builder. */ - public Builder setFindAndReplace(final Map> value) { + public Builder setFindAndReplace(final ImmutableMap> value) { _findAndReplace = value; return this; } @@ -377,11 +396,22 @@ public Builder setFindAndReplace(final Map> value * @param value List of dimensions to inject. * @return This instance of Builder. */ - public Builder setInject(final List value) { + public Builder setInject(final ImmutableMap value) { _inject = value; return this; } + /** + * Sets dimensions to remove. Optional. Cannot be null. Defaults to empty. + * + * @param value List of dimensions to inject. + * @return This instance of Builder. + */ + public Builder setRemove(final ImmutableList value) { + _remove = value; + return this; + } + @Override protected Builder self() { return this; @@ -390,8 +420,10 @@ protected Builder self() { @NotNull private Source _source; @NotNull - private Map> _findAndReplace = Maps.newHashMap(); + private ImmutableMap> _findAndReplace = ImmutableMap.of(); + @NotNull + private ImmutableMap _inject = ImmutableMap.of(); @NotNull - private List _inject = Lists.newArrayList(); + private ImmutableList _remove = ImmutableList.of(); } } diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index c0810bb6..3dbba012 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -28,7 +28,6 @@ import com.arpnetworking.tsdcore.model.Unit; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Lists; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Before; @@ -36,10 +35,7 @@ import org.mockito.ArgumentCaptor; import org.mockito.Mockito; -import java.util.Collections; -import java.util.List; import java.util.Map; -import java.util.regex.Pattern; /** * Tests for the MergingSource class. @@ -60,17 +56,6 @@ public void setUp() { "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))) - .setInject(Lists.newArrayList( - new TransformingSource.DimensionInjection.Builder() - .setDimension("inject") - .setValue("value") - .setReplaceExisting(true) - .build(), - new TransformingSource.DimensionInjection.Builder() - .setDimension("no_over") - .setValue("not_overwriting") - .setReplaceExisting(false) - .build())) .setSource(_mockSource); } @@ -104,12 +89,14 @@ public void testMergingObserverInvalidEvent() { final TransformingSource transformingSource = new TransformingSource.Builder() .setName("testMergingObserverInvalidEventTransformingSource") .setSource(_mockSource) - .setFindAndReplace(Collections.>emptyMap()) + .setFindAndReplace(ImmutableMap.of()) .build(); Mockito.reset(_mockSource); new TransformingSource.TransformingObserver( transformingSource, - Collections.>emptyMap()) + ImmutableMap.of(), + ImmutableMap.of(), + ImmutableList.of()) .notify(OBSERVABLE, "Not a Record"); Mockito.verifyZeroInteractions(_mockSource); } @@ -266,6 +253,31 @@ public void testReplaceWithCapture() { assertRecordsEqual(actualRecord, expectedRecord); } + @Test + public void testInjectsDimension() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(matchingRecord.getMetrics()) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + @Test public void testExtractTagWithCapture() { final Record matchingRecord = TestBeanFactory.createRecordBuilder() @@ -385,6 +397,154 @@ public void testMatchOverridesTagInCapture() { assertRecordsEqual(actualRecord, expectedRecord); } + @Test + public void testStaticDimensionInjection() { + _transformingSourceBuilder.setInject(ImmutableMap.of( + "injected", + new TransformingSource.DimensionInjection.Builder() + .setValue("value") + .setReplaceExisting(false) + .build())); + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("injected", "value"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testStaticDimensionInjectionOverwrite() { + _transformingSourceBuilder.setInject(ImmutableMap.of( + "injected", + new TransformingSource.DimensionInjection.Builder() + .setValue("new_value") + .setReplaceExisting(true) + .build(), + "injected_no_over", + new TransformingSource.DimensionInjection.Builder() + .setValue("new_value") + .setReplaceExisting(false) + .build())); + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "injected", "old_value", + "injected_no_over", "old_value")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("injected", "new_value"); + expectedDimensions.put("injected_no_over", "old_value"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testRemoveDimension() { + _transformingSourceBuilder.setRemove(ImmutableList.of("remove")); + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "remove", "_value")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.remove("remove"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + @Test public void testExtractOverridesExisting() { final Record matchingRecord = TestBeanFactory.createRecordBuilder() From d1a75f383c5a14cd215eb91c0604ce12a1bfe122 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Thu, 5 Apr 2018 11:51:21 -0700 Subject: [PATCH 04/11] PR updates --- .../mad/sources/TransformingSource.java | 45 ++++++++++--------- .../mad/sources/TransformingSourceTest.java | 4 +- 2 files changed, 27 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index ae379dcc..bee8746b 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -29,6 +29,8 @@ import com.arpnetworking.steno.LogValueMapFactory; import com.arpnetworking.steno.Logger; import com.arpnetworking.steno.LoggerFactory; +import com.arpnetworking.tsdcore.model.DefaultKey; +import com.arpnetworking.tsdcore.model.Key; import com.arpnetworking.tsdcore.model.MetricType; import com.arpnetworking.tsdcore.model.Quantity; import com.arpnetworking.utility.RegexAndMapReplacer; @@ -41,7 +43,6 @@ import net.sf.oval.constraint.NotNull; import java.util.Collections; -import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -90,9 +91,9 @@ private TransformingSource(final Builder builder) { super(builder); _source = builder._source; - final ImmutableMap.Builder> findReplaceBuilder = + final ImmutableMap.Builder> findReplaceBuilder = ImmutableMap.builderWithExpectedSize(builder._findAndReplace.size()); - for (final Map.Entry> entry : builder._findAndReplace.entrySet()) { + for (final ImmutableMap.Entry> entry : builder._findAndReplace.entrySet()) { findReplaceBuilder.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); } _findAndReplace = findReplaceBuilder.build(); @@ -104,7 +105,7 @@ private TransformingSource(final Builder builder) { } private final Source _source; - private final ImmutableMap> _findAndReplace; + private final ImmutableMap> _findAndReplace; private final ImmutableMap _inject; private final ImmutableList _remove; @@ -116,7 +117,7 @@ private TransformingSource(final Builder builder) { /* package private */ TransformingObserver( final TransformingSource source, - final Map> findAndReplace, + final ImmutableMap> findAndReplace, final ImmutableMap inject, final ImmutableList remove) { _source = source; @@ -137,11 +138,11 @@ public void notify(final Observable observable, final Object event) { // Merge the metrics in the record together final Record record = (Record) event; - final Map, Map> mergedMetrics = Maps.newHashMap(); + final Map> mergedMetrics = Maps.newHashMap(); for (final Map.Entry metric : record.getMetrics().entrySet()) { boolean found = false; final String metricName = metric.getKey(); - for (final Map.Entry> findAndReplace : _findAndReplace.entrySet()) { + for (final Map.Entry> findAndReplace : _findAndReplace.entrySet()) { final Pattern metricPattern = findAndReplace.getKey(); final Matcher matcher = metricPattern.matcher(metricName); if (matcher.find()) { @@ -158,7 +159,7 @@ public void notify(final Observable observable, final Object event) { metric.getValue(), replacedString, mergedMetrics, - getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), ImmutableList.of())); + getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), consumedDimensions)); } else { final String newMetricName = replacedString.substring(0, tagsStart); final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); @@ -184,7 +185,7 @@ public void notify(final Observable observable, final Object event) { // Raise the merged record event with this source's observers // NOTE: Do not leak instances of MergingMetric since it is mutable - for (Map.Entry, Map> entry : mergedMetrics.entrySet()) { + for (final Map.Entry> entry : mergedMetrics.entrySet()) { _source.notify( ThreadLocalBuilder.build( DefaultRecord.Builder.class, @@ -198,16 +199,15 @@ public void notify(final Observable observable, final Object event) { .setId(record.getId()) .setTime(record.getTime()) .setAnnotations(record.getAnnotations()) - .setDimensions(entry.getKey()))); + .setDimensions(entry.getKey().getParameters()))); } } - private ImmutableMap getModifiedDimensions( + private Key getModifiedDimensions( final ImmutableMap inputDimensions, final Map add, final ImmutableList remove) { - final Map finalTags = Maps.newHashMap(); - finalTags.putAll(inputDimensions); + final Map finalTags = Maps.newHashMap(inputDimensions); // Remove the dimensions that we consumed in the replacement remove.forEach(finalTags::remove); _remove.forEach(finalTags::remove); @@ -217,14 +217,17 @@ private ImmutableMap getModifiedDimensions( inject.isReplaceExisting() || oldValue == null ? inject.getValue() : oldValue)); finalTags.putAll(add); - return ImmutableMap.copyOf(finalTags); + return new DefaultKey(ImmutableMap.copyOf(finalTags)); } - private void merge(final Metric metric, final String key, - final Map, Map> mergedMetrics, - final ImmutableMap dimensions) { + private void merge( + final Metric metric, + final String key, + final Map> mergedMetrics, + final Key dimensionKey) { - final Map mergedMetricsForDimensions = mergedMetrics.computeIfAbsent(dimensions, k -> Maps.newHashMap()); + final Map mergedMetricsForDimensions = + mergedMetrics.computeIfAbsent(dimensionKey, k -> Maps.newHashMap()); final MergingMetric mergedMetric = mergedMetricsForDimensions.get(key); if (mergedMetric == null) { // This is the first time this metric is being merged into @@ -244,7 +247,7 @@ private void merge(final Metric metric, final String key, } private final TransformingSource _source; - private final Map> _findAndReplace; + private final ImmutableMap> _findAndReplace; private final ImmutableMap _inject; private final ImmutableList _remove; } @@ -385,7 +388,7 @@ public Builder setSource(final Source value) { * @param value The find and replace expression map. * @return This instance of Builder. */ - public Builder setFindAndReplace(final ImmutableMap> value) { + public Builder setFindAndReplace(final ImmutableMap> value) { _findAndReplace = value; return this; } @@ -420,7 +423,7 @@ protected Builder self() { @NotNull private Source _source; @NotNull - private ImmutableMap> _findAndReplace = ImmutableMap.of(); + private ImmutableMap> _findAndReplace = ImmutableMap.of(); @NotNull private ImmutableMap _inject = ImmutableMap.of(); @NotNull diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index 3dbba012..68e86595 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -335,11 +335,13 @@ public void testInjectTagWithCapture() { .build(); final Record actualRecord = mapRecord(matchingRecord); + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.remove("animal"); final Record expectedRecord = TestBeanFactory.createRecordBuilder() .setAnnotations(matchingRecord.getAnnotations()) .setTime(matchingRecord.getTime()) - .setDimensions(matchingRecord.getDimensions()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) .setMetrics(ImmutableMap.of( "tagged/frog/animal", TestBeanFactory.createMetricBuilder() From a4a0d37da324325a19971b66e96e0140ad8f06b7 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Sat, 7 Apr 2018 01:06:05 -0700 Subject: [PATCH 05/11] extract tranformation set --- .../mad/sources/TransformingSource.java | 278 ++++++++++-------- .../mad/sources/TransformingSourceTest.java | 16 +- 2 files changed, 172 insertions(+), 122 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index bee8746b..29e5433d 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -78,7 +78,7 @@ public void stop() { public Object toLogValue() { return LogValueMapFactory.builder(this) .put("source", _source) - .put("findAndReplace", _findAndReplace) + .put("tranformations", _transformations) .build(); } @@ -90,24 +90,13 @@ public String toString() { private TransformingSource(final Builder builder) { super(builder); _source = builder._source; + _transformations = builder._transformations; - final ImmutableMap.Builder> findReplaceBuilder = - ImmutableMap.builderWithExpectedSize(builder._findAndReplace.size()); - for (final ImmutableMap.Entry> entry : builder._findAndReplace.entrySet()) { - findReplaceBuilder.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); - } - _findAndReplace = findReplaceBuilder.build(); - - _inject = builder._inject; - _remove = builder._remove; - - _source.attach(new TransformingObserver(this, _findAndReplace, _inject, _remove)); + _source.attach(new TransformingObserver(this, _transformations)); } private final Source _source; - private final ImmutableMap> _findAndReplace; - private final ImmutableMap _inject; - private final ImmutableList _remove; + private final ImmutableList _transformations; private static final Logger LOGGER = LoggerFactory.getLogger(TransformingSource.class); private static final Splitter.MapSplitter TAG_SPLITTER = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('='); @@ -117,13 +106,9 @@ private TransformingSource(final Builder builder) { /* package private */ TransformingObserver( final TransformingSource source, - final ImmutableMap> findAndReplace, - final ImmutableMap inject, - final ImmutableList remove) { + final ImmutableList transformations) { _source = source; - _findAndReplace = findAndReplace; - _inject = inject; - _remove = remove; + _transformations = transformations; } @Override @@ -139,79 +124,86 @@ public void notify(final Observable observable, final Object event) { // Merge the metrics in the record together final Record record = (Record) event; final Map> mergedMetrics = Maps.newHashMap(); - for (final Map.Entry metric : record.getMetrics().entrySet()) { - boolean found = false; - final String metricName = metric.getKey(); - for (final Map.Entry> findAndReplace : _findAndReplace.entrySet()) { - final Pattern metricPattern = findAndReplace.getKey(); - final Matcher matcher = metricPattern.matcher(metricName); - if (matcher.find()) { - for (final String replacement : findAndReplace.getValue()) { - final RegexAndMapReplacer.Replacement rep = - RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, record.getDimensions()); - final String replacedString = rep.getReplacement(); - final ImmutableList consumedDimensions = rep.getVariablesMatched(); - - final int tagsStart = replacedString.indexOf(';'); - if (tagsStart == -1) { - // We just have a metric name. Optimize for this common case - merge( - metric.getValue(), - replacedString, - mergedMetrics, - getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), consumedDimensions)); - } else { - final String newMetricName = replacedString.substring(0, tagsStart); - final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); - merge( - metric.getValue(), - newMetricName, - mergedMetrics, - getModifiedDimensions(record.getDimensions(), parsedTags, consumedDimensions)); + for (TransformationSet transformation : _transformations) { + for (final Map.Entry metric : record.getMetrics().entrySet()) { + boolean found = false; + final String metricName = metric.getKey(); + for (final Map.Entry> findAndReplace : transformation.getFindAndReplace().entrySet()) { + final Pattern metricPattern = findAndReplace.getKey(); + final Matcher matcher = metricPattern.matcher(metricName); + if (matcher.find()) { + for (final String replacement : findAndReplace.getValue()) { + final RegexAndMapReplacer.Replacement rep = + RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, record.getDimensions()); + final String replacedString = rep.getReplacement(); + final ImmutableList consumedDimensions = rep.getVariablesMatched(); + + final int tagsStart = replacedString.indexOf(';'); + if (tagsStart == -1) { + // We just have a metric name. Optimize for this common case + merge( + metric.getValue(), + replacedString, + mergedMetrics, + getModifiedDimensions( + record.getDimensions(), + Collections.emptyMap(), + consumedDimensions, + transformation)); + } else { + final String newMetricName = replacedString.substring(0, tagsStart); + final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); + merge( + metric.getValue(), + newMetricName, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), parsedTags, consumedDimensions, transformation)); + } } + //Having "found" set here means that mapping a metric to an empty list suppresses that metric + found = true; } - //Having "found" set here means that mapping a metric to an empty list suppresses that metric - found = true; + } + if (!found) { + merge( + metric.getValue(), + metricName, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), ImmutableList.of(), transformation)); } } - if (!found) { - merge( - metric.getValue(), - metricName, - mergedMetrics, - getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), ImmutableList.of())); - } - } - // Raise the merged record event with this source's observers - // NOTE: Do not leak instances of MergingMetric since it is mutable - for (final Map.Entry> entry : mergedMetrics.entrySet()) { - _source.notify( - ThreadLocalBuilder.build( - DefaultRecord.Builder.class, - b1 -> b1.setMetrics( - entry.getValue().entrySet().stream().collect( - ImmutableMap.toImmutableMap( - Map.Entry::getKey, - e -> ThreadLocalBuilder.clone( - e.getValue(), - DefaultMetric.Builder.class)))) - .setId(record.getId()) - .setTime(record.getTime()) - .setAnnotations(record.getAnnotations()) - .setDimensions(entry.getKey().getParameters()))); + // Raise the merged record event with this source's observers + // NOTE: Do not leak instances of MergingMetric since it is mutable + for (final Map.Entry> entry : mergedMetrics.entrySet()) { + _source.notify( + ThreadLocalBuilder.build( + DefaultRecord.Builder.class, + b1 -> b1.setMetrics( + entry.getValue().entrySet().stream().collect( + ImmutableMap.toImmutableMap( + Map.Entry::getKey, + e -> ThreadLocalBuilder.clone( + e.getValue(), + DefaultMetric.Builder.class)))) + .setId(record.getId()) + .setTime(record.getTime()) + .setAnnotations(record.getAnnotations()) + .setDimensions(entry.getKey().getParameters()))); + } } } private Key getModifiedDimensions( final ImmutableMap inputDimensions, final Map add, - final ImmutableList remove) { + final ImmutableList remove, + final TransformationSet transformation) { final Map finalTags = Maps.newHashMap(inputDimensions); // Remove the dimensions that we consumed in the replacement remove.forEach(finalTags::remove); - _remove.forEach(finalTags::remove); - _inject.forEach( + transformation.getRemove().forEach(finalTags::remove); + transformation.getInject().forEach( (key, inject) -> finalTags.compute(key, (k, oldValue) -> inject.isReplaceExisting() || oldValue == null ? inject.getValue() : oldValue)); @@ -247,9 +239,7 @@ private void merge( } private final TransformingSource _source; - private final ImmutableMap> _findAndReplace; - private final ImmutableMap _inject; - private final ImmutableList _remove; + private final ImmutableList _transformations; } // NOTE: Package private for testing @@ -357,6 +347,87 @@ public Builder setReplaceExisting(final Boolean value) { } } + public static final class TransformationSet { + public ImmutableMap> getFindAndReplace() { + return _findAndReplace; + } + + public ImmutableMap getInject() { + return _inject; + } + + public ImmutableList getRemove() { + return _remove; + } + + private TransformationSet(final Builder builder) { + final ImmutableMap.Builder> findReplaceBuilder = + ImmutableMap.builderWithExpectedSize(builder._findAndReplace.size()); + for (final ImmutableMap.Entry> entry : builder._findAndReplace.entrySet()) { + findReplaceBuilder.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); + } + _findAndReplace = findReplaceBuilder.build(); + _inject = builder._inject; + _remove = builder._remove; + } + + private final ImmutableMap _inject; + private final ImmutableList _remove; + private final ImmutableMap> _findAndReplace; + + /** + * Implementation of the builder pattern for a {@link TransformationSet}. + */ + public static final class Builder extends OvalBuilder { + /** + * Public constructor. + */ + public Builder() { + super(TransformationSet::new); + } + + /** + * Sets find and replace expression map. Optional. Cannot be null. Defaults to empty. + * + * @param value The find and replace expression map. + * @return This instance of Builder. + */ + public Builder setFindAndReplace(final ImmutableMap> value) { + _findAndReplace = value; + return this; + } + + /** + * Sets dimensions to inject. Optional. Cannot be null. Defaults to empty. + * + * @param value List of dimensions to inject. + * @return This instance of Builder. + */ + public Builder setInject(final ImmutableMap value) { + _inject = value; + return this; + } + + /** + * Sets dimensions to remove. Optional. Cannot be null. Defaults to empty. + * + * @param value List of dimensions to inject. + * @return This instance of Builder. + */ + public Builder setRemove(final ImmutableList value) { + _remove = value; + return this; + } + + @NotNull + private ImmutableMap> _findAndReplace = ImmutableMap.of(); + @NotNull + private ImmutableMap _inject = ImmutableMap.of(); + @NotNull + private ImmutableList _remove = ImmutableList.of(); + } + } + /** * Implementation of builder pattern for {@link TransformingSource}. * @@ -375,7 +446,7 @@ public Builder() { * Sets the underlying source. Cannot be null. * * @param value The underlying source. - * @return This instance of Builder. + * @return This instance of {@link Builder}. */ public Builder setSource(final Source value) { _source = value; @@ -383,35 +454,13 @@ public Builder setSource(final Source value) { } /** - * Sets find and replace expression map. Optional. Cannot be null. Defaults to empty. - * - * @param value The find and replace expression map. - * @return This instance of Builder. - */ - public Builder setFindAndReplace(final ImmutableMap> value) { - _findAndReplace = value; - return this; - } - - /** - * Sets dimensions to inject. Optional. Cannot be null. Defaults to empty. + * Sets the transformations. Required. Cannot be null. Cannot be empty. * - * @param value List of dimensions to inject. - * @return This instance of Builder. + * @param value The list of transformations to apply. + * @return This instance of {@link Builder}. */ - public Builder setInject(final ImmutableMap value) { - _inject = value; - return this; - } - - /** - * Sets dimensions to remove. Optional. Cannot be null. Defaults to empty. - * - * @param value List of dimensions to inject. - * @return This instance of Builder. - */ - public Builder setRemove(final ImmutableList value) { - _remove = value; + public Builder setTransformations(final ImmutableList value) { + _transformations = value; return this; } @@ -423,10 +472,7 @@ protected Builder self() { @NotNull private Source _source; @NotNull - private ImmutableMap> _findAndReplace = ImmutableMap.of(); - @NotNull - private ImmutableMap _inject = ImmutableMap.of(); - @NotNull - private ImmutableList _remove = ImmutableList.of(); + @NotEmpty + private ImmutableList _transformations; } } diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index 68e86595..cb860aaf 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -50,12 +50,16 @@ public void setUp() { _mockSource = Mockito.mock(Source.class); _transformingSourceBuilder = new TransformingSource.Builder() .setName("TransformingSourceTest") - .setFindAndReplace(ImmutableMap.of( - "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), - "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), - "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), - "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), - "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))) + .setTransformations(ImmutableList.of( + new TransformingSource.TransformationSet.Builder() + .setFindAndReplace(ImmutableMap.of( + "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), + "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), + "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), + "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), + "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))) + .build() + )) .setSource(_mockSource); } From 0828bd6ba03400a4409ca8e0364f6c05699d1295 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Sat, 7 Apr 2018 23:28:13 -0700 Subject: [PATCH 06/11] fix tests --- .../mad/sources/TransformingSource.java | 5 +++ .../mad/sources/TransformingSourceTest.java | 36 ++++++++++--------- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index 29e5433d..ac8b433e 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -347,6 +347,11 @@ public Builder setReplaceExisting(final Boolean value) { } } + /** + * Represents a set of transformations to apply. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ public static final class TransformationSet { public ImmutableMap> getFindAndReplace() { return _findAndReplace; diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index cb860aaf..57c3d98b 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -48,18 +48,16 @@ public class TransformingSourceTest { public void setUp() { _mockObserver = Mockito.mock(Observer.class); _mockSource = Mockito.mock(Source.class); + _transformSetBuilder = new TransformingSource.TransformationSet.Builder() + .setFindAndReplace(ImmutableMap.of( + "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), + "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), + "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), + "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), + "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))); _transformingSourceBuilder = new TransformingSource.Builder() .setName("TransformingSourceTest") - .setTransformations(ImmutableList.of( - new TransformingSource.TransformationSet.Builder() - .setFindAndReplace(ImmutableMap.of( - "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), - "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), - "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), - "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), - "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))) - .build() - )) + .setTransformations(ImmutableList.of(_transformSetBuilder.build())) .setSource(_mockSource); } @@ -90,17 +88,19 @@ public void testToString() { @Test public void testMergingObserverInvalidEvent() { + final TransformingSource.TransformationSet transformationSet = new TransformingSource.TransformationSet.Builder() + .setFindAndReplace(ImmutableMap.of()) + .build(); final TransformingSource transformingSource = new TransformingSource.Builder() .setName("testMergingObserverInvalidEventTransformingSource") .setSource(_mockSource) - .setFindAndReplace(ImmutableMap.of()) + .setTransformations(ImmutableList.of( + transformationSet)) .build(); Mockito.reset(_mockSource); new TransformingSource.TransformingObserver( transformingSource, - ImmutableMap.of(), - ImmutableMap.of(), - ImmutableList.of()) + ImmutableList.of(transformationSet)) .notify(OBSERVABLE, "Not a Record"); Mockito.verifyZeroInteractions(_mockSource); } @@ -405,7 +405,7 @@ public void testMatchOverridesTagInCapture() { @Test public void testStaticDimensionInjection() { - _transformingSourceBuilder.setInject(ImmutableMap.of( + _transformSetBuilder.setInject(ImmutableMap.of( "injected", new TransformingSource.DimensionInjection.Builder() .setValue("value") @@ -453,7 +453,7 @@ public void testStaticDimensionInjection() { @Test public void testStaticDimensionInjectionOverwrite() { - _transformingSourceBuilder.setInject(ImmutableMap.of( + _transformSetBuilder.setInject(ImmutableMap.of( "injected", new TransformingSource.DimensionInjection.Builder() .setValue("new_value") @@ -509,7 +509,7 @@ public void testStaticDimensionInjectionOverwrite() { @Test public void testRemoveDimension() { - _transformingSourceBuilder.setRemove(ImmutableList.of("remove")); + _transformSetBuilder.setRemove(ImmutableList.of("remove")); final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( "doesnt_match", @@ -697,6 +697,7 @@ private void assertRecordsEqual(final Record actualRecord, final Record expected } private Record mapRecord(final Record record) { + _transformingSourceBuilder.setTransformations(ImmutableList.of(_transformSetBuilder.build())); final Source transformingSource = _transformingSourceBuilder.build(); transformingSource.attach(_mockObserver); notify(_mockSource, record); @@ -717,6 +718,7 @@ private static void notify(final Observable observable, final Object event) { private Observer _mockObserver; private Source _mockSource; private TransformingSource.Builder _transformingSourceBuilder; + private TransformingSource.TransformationSet.Builder _transformSetBuilder; private static final Observable OBSERVABLE = new Observable() { @Override From 05954a4565d9e791c811397b9061bfb5beb3bdf7 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Tue, 10 Apr 2018 16:11:33 -0700 Subject: [PATCH 07/11] added prefix support and precedence --- .../mad/sources/TransformingSource.java | 16 ++- .../utility/RegexAndMapReplacer.java | 102 ++++++++++++++++-- .../utility/RegexAndMapBenchmarkTest.java | 4 +- .../utility/RegexAndMapReplacerTest.java | 88 +++++++++++++-- 4 files changed, 188 insertions(+), 22 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index ac8b433e..d46e5229 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -43,9 +43,12 @@ import net.sf.oval.constraint.NotNull; import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * Implementation of {@link Source} which wraps another {@link Source} @@ -124,6 +127,10 @@ public void notify(final Observable observable, final Object event) { // Merge the metrics in the record together final Record record = (Record) event; final Map> mergedMetrics = Maps.newHashMap(); + final LinkedHashMap> variablesMap = new LinkedHashMap<>(); + variablesMap.put("dimension", record.getDimensions()); + variablesMap.put("env", System.getenv()); + for (TransformationSet transformation : _transformations) { for (final Map.Entry metric : record.getMetrics().entrySet()) { boolean found = false; @@ -134,9 +141,12 @@ public void notify(final Observable observable, final Object event) { if (matcher.find()) { for (final String replacement : findAndReplace.getValue()) { final RegexAndMapReplacer.Replacement rep = - RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, record.getDimensions()); + RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, variablesMap); final String replacedString = rep.getReplacement(); - final ImmutableList consumedDimensions = rep.getVariablesMatched(); + final List consumedDimensions = rep.getVariablesMatched().stream() + .filter(var -> var.startsWith("dimension:") || var.indexOf(':') == -1) // Only dimension vars + .map(var -> var.substring(var.indexOf(":") + 1)) // Strip the prefix + .collect(Collectors.toList()); final int tagsStart = replacedString.indexOf(';'); if (tagsStart == -1) { @@ -197,7 +207,7 @@ public void notify(final Observable observable, final Object event) { private Key getModifiedDimensions( final ImmutableMap inputDimensions, final Map add, - final ImmutableList remove, + final List remove, final TransformationSet transformation) { final Map finalTags = Maps.newHashMap(inputDimensions); // Remove the dimensions that we consumed in the replacement diff --git a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java index bbc72487..200ca13f 100644 --- a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java +++ b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java @@ -16,8 +16,8 @@ package com.arpnetworking.utility; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; +import java.util.LinkedHashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -43,11 +43,11 @@ public final class RegexAndMapReplacer { * @param variables map of variables to include * @return a string with replacement tokens replaced */ - public static Replacement replaceAll( + public static Replacement replaceAll( final Pattern pattern, final String input, final String replace, - final ImmutableMap variables) { + final LinkedHashMap> variables) { final Matcher matcher = pattern.matcher(input); boolean found = matcher.find(); if (found) { @@ -73,7 +73,7 @@ private static void appendReplacement( final Matcher matcher, final String replacement, final StringBuilder replacementBuilder, - final Map variables, + final LinkedHashMap> variables, final ImmutableList.Builder variablesUsedBuilder) { final StringBuilder tokenBuilder = new StringBuilder(); int x = -1; @@ -112,7 +112,7 @@ private static int writeReplacementToken( final int offset, final StringBuilder output, final Matcher matcher, - final Map variables, + final LinkedHashMap> variables, final StringBuilder tokenBuilder, final ImmutableList.Builder variablesUsedBuilder) { boolean inReplaceBrackets = false; @@ -175,25 +175,105 @@ private static String getReplacement( final Matcher matcher, final String replaceToken, final boolean numeric, - final Map variables, + final LinkedHashMap> variables, final ImmutableList.Builder variablesUsedBuilder) { if (numeric) { final int replaceGroup = Integer.parseInt(replaceToken); return matcher.group(replaceGroup); } else { - try { - return matcher.group(replaceToken); - } catch (final IllegalArgumentException e) { // No group with this name - variablesUsedBuilder.add(replaceToken); - return variables.getOrDefault(replaceToken, ""); + final int prefixSeparatorIndex = replaceToken.indexOf(':'); + final String prefix; + final String variableName; + if (prefixSeparatorIndex == replaceToken.length() - 1) { + throw new IllegalArgumentException( + String.format("found prefix in variable replacement, but no variable name found: '%s'", + replaceToken)); + } + if (prefixSeparatorIndex != -1) { + prefix = replaceToken.substring(0, prefixSeparatorIndex); + variableName = replaceToken.substring(prefixSeparatorIndex + 1); + } else { + prefix = ""; + variableName = replaceToken; + } + if (prefix.isEmpty()) { + return replaceNonPrefix(matcher, variables, variablesUsedBuilder, variableName); + + } else { + return replacePrefix(matcher, variables, variablesUsedBuilder, prefix, variableName); } } } + private static String replacePrefix( + final Matcher matcher, + final LinkedHashMap> variables, + final ImmutableList.Builder variablesUsedBuilder, + final String prefix, + final String variableName) { + if (prefix.equals("capture")) { + if (isNumeric(variableName)) { + final Integer groupIndex = Integer.valueOf(variableName); + return matcher.group(groupIndex); + } else { + try { + return matcher.group(variableName); + } catch (final IllegalArgumentException ignored2) { // No group with this name + return ""; + } + } + } + + // Only record variables that are not captures + variablesUsedBuilder.add(String.format("%s:%s", prefix, variableName)); + + final Map variableMap = variables.get(prefix); + if (variableMap == null) { + throw new IllegalArgumentException(String.format("could not find map for variables with prefix '%s'", prefix)); + } + return variableMap.getOrDefault(variableName, ""); + } + + private static String replaceNonPrefix( + final Matcher matcher, + final LinkedHashMap> variables, + final ImmutableList.Builder variablesUsedBuilder, + final String variableName) { + // First try the capture group with the name + try { + return matcher.group(variableName); + } catch (final IllegalArgumentException e) { // No group with this name + // Walk through the variable maps in order to find the first match + for (final Map.Entry> entry : variables.entrySet()) { + final Map variableMap = entry.getValue(); + final String replacement = variableMap.get(variableName); + if (replacement != null) { + variablesUsedBuilder.add(String.format("%s:%s", entry.getKey(), variableName)); + return replacement; + } + } + } + return ""; + } + + private static boolean isNumeric(final String string) { + boolean isNumeric = true; + for (int x = 0; x < string.length(); x++) { + if (!Character.isDigit(string.charAt(x))) { + isNumeric = false; + break; + } + } + return isNumeric; + } + private RegexAndMapReplacer() { } /** * Describes the replacement string and variables used in it's creation. + * + * The "replacement" field is the resulting string. + * The "variablesMatched" field is a list of input variables that were matched, in prefix:variable form */ public static final class Replacement { public String getReplacement() { diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java index b9b18dc9..969e2bee 100644 --- a/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java @@ -17,11 +17,11 @@ import com.carrotsearch.junitbenchmarks.BenchmarkOptions; import com.carrotsearch.junitbenchmarks.BenchmarkRule; -import com.google.common.collect.ImmutableMap; import org.junit.Rule; import org.junit.Test; import org.junit.rules.TestRule; +import java.util.LinkedHashMap; import java.util.regex.Pattern; /** @@ -34,7 +34,7 @@ public final class RegexAndMapBenchmarkTest { @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) @Test public void testRegexAndMap() { - final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, ImmutableMap.of()).getReplacement(); + final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, new LinkedHashMap<>()).getReplacement(); } @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java index 4c56cd8b..fa50c6aa 100644 --- a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java @@ -15,10 +15,13 @@ */ package com.arpnetworking.utility; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.junit.Assert; import org.junit.Test; +import java.util.LinkedHashMap; +import java.util.Map; import java.util.regex.Pattern; /** @@ -41,7 +44,7 @@ public void testInvalidEscape() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "${\\avariable}"; // \a is an invalid escape sequence - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } @Test(expected = IllegalArgumentException.class) @@ -49,7 +52,7 @@ public void testMissingClosingCurly() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "${0"; // no ending } - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } @Test(expected = IllegalArgumentException.class) @@ -57,7 +60,15 @@ public void testInvalidEscapeAtEnd() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "${0}\\"; // trailing \ - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidNoVariableName() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${prefix:}"; // variable prefix, but no name + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } @Test @@ -74,7 +85,7 @@ public void testInvalidReplacementTokenMissingOpen() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; final String replace = "$variable"; // replacement variable has no { - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, ImmutableMap.of()).getReplacement(); + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } @Test @@ -167,6 +178,54 @@ public void testSingleMatchPartialMultipleGroupNameOverridesVariablesReplace() { testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "bad", "g2", "value")); } + @Test + public void testMatchPrefix() { + final Pattern pattern = Pattern.compile("input_string_(?.*)_(.*)"); + final String input = "input_string_ending_ending2"; + final String replace = "${prefix1:var}_${prefix2:var}_${capture:var}_${capture:2}"; + final String expected = "var1_var2_ending_ending2"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "var1")); + variables.put("prefix2", ImmutableMap.of("var", "var2")); + final RegexAndMapReplacer.Replacement replacement = testExpression(pattern, input, replace, expected, variables); + Assert.assertEquals(ImmutableList.of("prefix1:var", "prefix2:var"), replacement.getVariablesMatched()); + } + + @Test + public void testMatchPrecedence() { + final Pattern pattern = Pattern.compile("input_string_(?.*)"); + final String input = "input_string_ending"; + final String replace = "${var}_${var2}_${var3}"; + final String expected = "ending_1-var2_2-var3"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "1-var", "var2", "1-var2")); + variables.put("prefix2", ImmutableMap.of("var", "2-var", "var2", "2-var2", "var3", "2-var3")); + final RegexAndMapReplacer.Replacement replacement = testExpression(pattern, input, replace, expected, variables); + Assert.assertEquals(ImmutableList.of("prefix1:var2", "prefix2:var3"), replacement.getVariablesMatched()); + } + + @Test + public void testMatchMissingVars() { + final Pattern pattern = Pattern.compile("input_string"); + final String input = "input_string_ending"; + final String replace = "${prefix1:missing}1${capture:missing}2${missing}"; + final String expected = "12_ending"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "var1")); + testExpression(pattern, input, replace, expected, variables); + } + + @Test(expected = IllegalArgumentException.class) + public void testMatchInvalidPrefix() { + final Pattern pattern = Pattern.compile("input_string_(?.*)"); + final String input = "input_string_ending"; + final String replace = "${notfound:var}_value"; + final String expected = "illegal arg exception"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "var1")); + testExpression(pattern, input, replace, expected, variables); + } + @Test public void testMultipleMatchFullStaticReplace() { final Pattern pattern = Pattern.compile("test"); @@ -185,13 +244,30 @@ public void testMultipleMatchPartialStaticReplace() { testExpression(pattern, input, replace, expected, ImmutableMap.of()); } - private void testExpression(final Pattern pattern, final String input, final String replace, final String expected, + private RegexAndMapReplacer.Replacement testExpression( + final Pattern pattern, + final String input, + final String replace, + final String expected, final ImmutableMap variables) { - final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables).getReplacement(); + final LinkedHashMap> variablesMap = new LinkedHashMap<>(); + variablesMap.put("myvars", variables); + return testExpression(pattern, input, replace, expected, variablesMap); + } + + private RegexAndMapReplacer.Replacement testExpression( + final Pattern pattern, + final String input, + final String replace, + final String expected, + final LinkedHashMap> variables) { + final RegexAndMapReplacer.Replacement replacement = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables); + final String result = replacement.getReplacement(); Assert.assertEquals(expected, result); try { final String stockResult = pattern.matcher(input).replaceAll(replace); Assert.assertEquals(expected, stockResult); } catch (final IllegalArgumentException ignored) { } + return replacement; } } From 9f3caed41f3fc366553b05e9a057dc7f29485d7c Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Tue, 10 Apr 2018 16:14:13 -0700 Subject: [PATCH 08/11] change replaceExisting to overwriteExisting --- .../metrics/mad/sources/TransformingSource.java | 16 ++++++++-------- .../mad/sources/TransformingSourceTest.java | 6 +++--- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index d46e5229..f823a42b 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -216,7 +216,7 @@ private Key getModifiedDimensions( transformation.getInject().forEach( (key, inject) -> finalTags.compute(key, (k, oldValue) -> - inject.isReplaceExisting() || oldValue == null ? inject.getValue() : oldValue)); + inject.isOverwriteExisting() || oldValue == null ? inject.getValue() : oldValue)); finalTags.putAll(add); return new DefaultKey(ImmutableMap.copyOf(finalTags)); @@ -302,17 +302,17 @@ public String getValue() { return _value; } - public boolean isReplaceExisting() { - return _replaceExisting; + public boolean isOverwriteExisting() { + return _overwriteExisting; } private DimensionInjection(final Builder builder) { _value = builder._value; - _replaceExisting = builder._replaceExisting; + _overwriteExisting = builder._overwriteExisting; } private final String _value; - private final boolean _replaceExisting; + private final boolean _overwriteExisting; /** * Implementation of the Builder pattern for {@link DimensionInjection}. @@ -344,8 +344,8 @@ public Builder setValue(final String value) { * @param value true to replace existing dimension value * @return This instance of {@link Builder}. */ - public Builder setReplaceExisting(final Boolean value) { - _replaceExisting = value; + public Builder setOverwriteExisting(final Boolean value) { + _overwriteExisting = value; return this; } @@ -353,7 +353,7 @@ public Builder setReplaceExisting(final Boolean value) { @NotEmpty private String _value; @NotNull - private Boolean _replaceExisting = true; + private Boolean _overwriteExisting = true; } } diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index 57c3d98b..56cb63ec 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -409,7 +409,7 @@ public void testStaticDimensionInjection() { "injected", new TransformingSource.DimensionInjection.Builder() .setValue("value") - .setReplaceExisting(false) + .setOverwriteExisting(false) .build())); final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( @@ -457,12 +457,12 @@ public void testStaticDimensionInjectionOverwrite() { "injected", new TransformingSource.DimensionInjection.Builder() .setValue("new_value") - .setReplaceExisting(true) + .setOverwriteExisting(true) .build(), "injected_no_over", new TransformingSource.DimensionInjection.Builder() .setValue("new_value") - .setReplaceExisting(false) + .setOverwriteExisting(false) .build())); final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( From 0e49ef7eadba63318333407dd258beb780311db5 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Tue, 10 Apr 2018 17:27:43 -0700 Subject: [PATCH 09/11] add some documentation --- README.md | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 74 insertions(+) diff --git a/README.md b/README.md index 81ca9271..adad4fc4 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,80 @@ interval="10s" [[inputs.kernel]] ``` +#### TransformSource +Sometimes you'll want to transform the incoming data before aggregating. +As shown in the example collectd source, it is possible to wrap a source with +another source that will transform the data. The TransformingSource specifies +sets of transformations to apply to the input records, allowing you to add dimensions +to all metrics, remove dimensions from all metrics, and modify metrics based on their +name. + +ex: +```hocon +{ + type="com.arpnetworking.metrics.mad.sources.TransformingSource" + name="transforming_source" + transformations = [ + { + inject = { + foo { + value = bar + overwriteExisting = false + } + } + remove = [ + baz + ] + findAndReplace = { + "this" = ["that"] + "extract/([^/]*)/thing" = ["extract/thing/${other_dimension};my_dimension=${1}"] + } + } + ] + source { + // your wrapped source goes here + } +} +``` + +tranformations is a list of TransformationSet objects. Each TransformationSet has an inject (Map\), + remove (List\) and a findAndReplace (Map\\>). + + A DimensionInjection is just a value and a boolean of whether or not to overwrite existing values. + + The keys in findAndReplace are regular expressions used to match metrics. If matched, the list of replacements is + executed, allowing for a single input metric to be recorded multiple times with different names or dimensions. + The format for the values in the list are similar to a standard java regex replacement. Variables are enclosed in ${}, + for example: + ```bash +${my_variable} +``` +Variables will be matched in the following order (first match wins): +1. capture group +2. dimension +3. environment variable + + +Variable names may also have a prefix that specifies where the value should originate. For example: +```bash +${capture:my_var} +``` +```bash +${env:MYVAR} +``` +```bash +${dimension:some_dimension} +``` + +This namespacing prevents the built-in precendence search for a matching variable. + +__Note: Any dimensions matched as part of a replacement will be removed from the resulting metric.__ + +Dimensions can also be injected into a metric. To do this, add a ';' after the replacement name of the metric +and proceed to specify key=value pairs (where both the key and value can use variables). Basically, the +replacement is processed, the string split on ';' with the first part being the metric name and anything +after the ';' being parsed as key=value pairs to be added to the metric's dimensions. + Development ----------- From 3c0f5aecb44066e9f948723c1e364c78816aa9a7 Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Fri, 13 Apr 2018 10:45:47 -0700 Subject: [PATCH 10/11] remove $n replacements, numerics to normal precedence --- README.md | 2 +- .../mad/sources/TransformingSource.java | 7 +- .../utility/RegexAndMapReplacer.java | 124 +++++++++--------- .../mad/sources/TransformingSourceTest.java | 4 +- .../utility/RegexAndMapReplacerTest.java | 45 +++++-- 5 files changed, 100 insertions(+), 82 deletions(-) diff --git a/README.md b/README.md index adad4fc4..f1facec2 100644 --- a/README.md +++ b/README.md @@ -324,7 +324,7 @@ ex: tranformations is a list of TransformationSet objects. Each TransformationSet has an inject (Map\), remove (List\) and a findAndReplace (Map\\>). - A DimensionInjection is just a value and a boolean of whether or not to overwrite existing values. + A DimensionInjection is just a value and a boolean of whether or not to overwrite existing values. Of omitted, overwrite defaults to true. The keys in findAndReplace are regular expressions used to match metrics. If matched, the list of replacements is executed, allowing for a single input metric to be recorded multiple times with different names or dimensions. diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index f823a42b..4eb21804 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -48,7 +48,6 @@ import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; -import java.util.stream.Collectors; /** * Implementation of {@link Source} which wraps another {@link Source} @@ -143,10 +142,8 @@ public void notify(final Observable observable, final Object event) { final RegexAndMapReplacer.Replacement rep = RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, variablesMap); final String replacedString = rep.getReplacement(); - final List consumedDimensions = rep.getVariablesMatched().stream() - .filter(var -> var.startsWith("dimension:") || var.indexOf(':') == -1) // Only dimension vars - .map(var -> var.substring(var.indexOf(":") + 1)) // Strip the prefix - .collect(Collectors.toList()); + final List consumedDimensions = rep.getVariablesMatched() + .getOrDefault("dimension", ImmutableList.of()); final int tagsStart = replacedString.indexOf(';'); if (tagsStart == -1) { diff --git a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java index 200ca13f..9f865887 100644 --- a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java +++ b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java @@ -16,16 +16,18 @@ package com.arpnetworking.utility; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; import java.util.LinkedHashMap; import java.util.Map; import java.util.regex.Matcher; import java.util.regex.Pattern; +import java.util.stream.Collectors; /** * A regex replacement utility that can also replace tokens not found in the regex. * - * $n where n is a number in the replace string is replaced by the pattern's match group n * ${name} in the replace string is replaced by the pattern's named capture, or by the value of the variable * with that name from the variables map * \ is used as an escape character and may be used to escape '$', '{', and '}' characters so that they will @@ -43,7 +45,7 @@ public final class RegexAndMapReplacer { * @param variables map of variables to include * @return a string with replacement tokens replaced */ - public static Replacement replaceAll( + public static Replacement replaceAll( final Pattern pattern, final String input, final String replace, @@ -51,22 +53,28 @@ public static Replacement replaceAll( final Matcher matcher = pattern.matcher(input); boolean found = matcher.find(); if (found) { - final ImmutableList.Builder variablesUsedBuilder = ImmutableList.builder(); + final Map> variablesUsed = Maps.newHashMap(); final StringBuilder builder = new StringBuilder(); int lastMatchedIndex = 0; do { builder.append(input.substring(lastMatchedIndex, matcher.start())); lastMatchedIndex = matcher.end(); - appendReplacement(matcher, replace, builder, variables, variablesUsedBuilder); + appendReplacement(matcher, replace, builder, variables, variablesUsed); found = matcher.find(); } while (found); // Append left-over string after the matches if (lastMatchedIndex < input.length() - 1) { builder.append(input.substring(lastMatchedIndex, input.length())); } - return new Replacement(builder.toString(), variablesUsedBuilder.build()); + final ImmutableMap> immutableVars = variablesUsed.entrySet() + .stream() + .collect( + Collectors.collectingAndThen( + Collectors.toMap(ImmutableMap.Entry::getKey, entry -> entry.getValue().build()), + ImmutableMap::copyOf)); + return new Replacement(builder.toString(), immutableVars); } - return new Replacement(input, ImmutableList.of()); + return new Replacement(input, ImmutableMap.of()); } private static void appendReplacement( @@ -74,7 +82,7 @@ private static void appendReplacement( final String replacement, final StringBuilder replacementBuilder, final LinkedHashMap> variables, - final ImmutableList.Builder variablesUsedBuilder) { + final Map> variablesUsed) { final StringBuilder tokenBuilder = new StringBuilder(); int x = -1; while (x < replacement.length() - 1) { @@ -85,7 +93,7 @@ private static void appendReplacement( processEscapedCharacter(replacement, x, replacementBuilder); } else { if (c == '$') { - x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder, variablesUsedBuilder); + x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder, variablesUsed); } else { replacementBuilder.append(c); } @@ -114,7 +122,7 @@ private static int writeReplacementToken( final Matcher matcher, final LinkedHashMap> variables, final StringBuilder tokenBuilder, - final ImmutableList.Builder variablesUsedBuilder) { + final Map> variablesUsed) { boolean inReplaceBrackets = false; boolean tokenNumeric = true; tokenBuilder.setLength(0); // reset the shared builder @@ -147,26 +155,13 @@ private static int writeReplacementToken( throw new IllegalArgumentException("Invalid replacement token, expected '}' at col " + x + ": " + replacement); } x++; // Consume the } - output.append(getReplacement(matcher, tokenBuilder.toString(), tokenNumeric, variables, variablesUsedBuilder)); + output.append(getReplacement(matcher, tokenBuilder.toString(), variables, variablesUsed)); } else { - // Consume until we hit a non-digit character - while (x < replacement.length()) { - c = replacement.charAt(x); - if (Character.isDigit(c)) { - tokenBuilder.append(c); - } else { - break; - } - x++; - } - if (tokenBuilder.length() == 0) { throw new IllegalArgumentException( String.format( - "Invalid replacement token, non-numeric tokens must be surrounded by { } at col %d: %s", + "Invalid replacement token, tokens must be surrounded by { } at col %d: %s", x, replacement)); - } - output.append(getReplacement(matcher, tokenBuilder.toString(), true, variables, variablesUsedBuilder)); } return x - offset - 1; } @@ -174,58 +169,54 @@ private static int writeReplacementToken( private static String getReplacement( final Matcher matcher, final String replaceToken, - final boolean numeric, final LinkedHashMap> variables, - final ImmutableList.Builder variablesUsedBuilder) { - if (numeric) { - final int replaceGroup = Integer.parseInt(replaceToken); - return matcher.group(replaceGroup); + final Map> variablesUsed) { + final int prefixSeparatorIndex = replaceToken.indexOf(':'); + final String prefix; + final String variableName; + if (prefixSeparatorIndex == replaceToken.length() - 1) { + throw new IllegalArgumentException( + String.format("found prefix in variable replacement, but no variable name found: '%s'", + replaceToken)); + } + if (prefixSeparatorIndex != -1) { + prefix = replaceToken.substring(0, prefixSeparatorIndex); + variableName = replaceToken.substring(prefixSeparatorIndex + 1); } else { - final int prefixSeparatorIndex = replaceToken.indexOf(':'); - final String prefix; - final String variableName; - if (prefixSeparatorIndex == replaceToken.length() - 1) { - throw new IllegalArgumentException( - String.format("found prefix in variable replacement, but no variable name found: '%s'", - replaceToken)); - } - if (prefixSeparatorIndex != -1) { - prefix = replaceToken.substring(0, prefixSeparatorIndex); - variableName = replaceToken.substring(prefixSeparatorIndex + 1); - } else { - prefix = ""; - variableName = replaceToken; - } - if (prefix.isEmpty()) { - return replaceNonPrefix(matcher, variables, variablesUsedBuilder, variableName); + prefix = ""; + variableName = replaceToken; + } + if (prefix.isEmpty()) { + return replaceNonPrefix(matcher, variables, variablesUsed, variableName); - } else { - return replacePrefix(matcher, variables, variablesUsedBuilder, prefix, variableName); - } + } else { + return replacePrefix(matcher, variables, variablesUsed, prefix, variableName); } } private static String replacePrefix( final Matcher matcher, final LinkedHashMap> variables, - final ImmutableList.Builder variablesUsedBuilder, + final Map> variablesUsed, final String prefix, final String variableName) { if (prefix.equals("capture")) { if (isNumeric(variableName)) { final Integer groupIndex = Integer.valueOf(variableName); - return matcher.group(groupIndex); - } else { - try { - return matcher.group(variableName); - } catch (final IllegalArgumentException ignored2) { // No group with this name - return ""; + if (groupIndex >= 0 && groupIndex <= matcher.groupCount()) { + return matcher.group(groupIndex); } } + + try { + return matcher.group(variableName); + } catch (final IllegalArgumentException ignored) { // No group with this name + return ""; + } } // Only record variables that are not captures - variablesUsedBuilder.add(String.format("%s:%s", prefix, variableName)); + variablesUsed.computeIfAbsent(prefix, key -> ImmutableList.builder()).add(variableName); final Map variableMap = variables.get(prefix); if (variableMap == null) { @@ -237,9 +228,16 @@ private static String replacePrefix( private static String replaceNonPrefix( final Matcher matcher, final LinkedHashMap> variables, - final ImmutableList.Builder variablesUsedBuilder, + final Map> variablesUsed, final String variableName) { - // First try the capture group with the name + // First try to check against the capture group number + if (isNumeric(variableName)) { + final int replaceGroup = Integer.parseInt(variableName); + if (replaceGroup >= 0 && replaceGroup <= matcher.groupCount()) { + return matcher.group(replaceGroup); + } + } + // Then try the capture group with the name try { return matcher.group(variableName); } catch (final IllegalArgumentException e) { // No group with this name @@ -248,7 +246,7 @@ private static String replaceNonPrefix( final Map variableMap = entry.getValue(); final String replacement = variableMap.get(variableName); if (replacement != null) { - variablesUsedBuilder.add(String.format("%s:%s", entry.getKey(), variableName)); + variablesUsed.computeIfAbsent(entry.getKey(), key -> ImmutableList.builder()).add(variableName); return replacement; } } @@ -280,17 +278,17 @@ public String getReplacement() { return _replacement; } - public ImmutableList getVariablesMatched() { + public ImmutableMap> getVariablesMatched() { return _variablesMatched; } - private Replacement(final String replacement, final ImmutableList variablesMatched) { + private Replacement(final String replacement, final ImmutableMap> variablesMatched) { _replacement = replacement; _variablesMatched = variablesMatched; } private final String _replacement; - private final ImmutableList _variablesMatched; + private final ImmutableMap> _variablesMatched; } } diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index 56cb63ec..b81e16f1 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -51,8 +51,8 @@ public void setUp() { _transformSetBuilder = new TransformingSource.TransformationSet.Builder() .setFindAndReplace(ImmutableMap.of( "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), - "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/$1"), - "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=$1"), + "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/${1}"), + "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=${1}"), "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))); _transformingSourceBuilder = new TransformingSource.Builder() diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java index fa50c6aa..450967b2 100644 --- a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java @@ -71,20 +71,19 @@ public void testInvalidNoVariableName() { final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } - @Test - public void testNumericWithClosingCurly() { + @Test(expected = IllegalArgumentException.class) + public void testInvalidReplacementTokenMissingOpen() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; - final String replace = "$0}"; - final String expected = "test}"; - testExpression(pattern, input, replace, expected, ImmutableMap.of()); + final String replace = "$variable"; // replacement variable has no { + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } @Test(expected = IllegalArgumentException.class) - public void testInvalidReplacementTokenMissingOpen() { + public void testInvalidReplacementTokenNumeric() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; - final String replace = "$variable"; // replacement variable has no { + final String replace = "$0"; // replacement variable has no { final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); } @@ -92,7 +91,7 @@ public void testInvalidReplacementTokenMissingOpen() { public void testGroup0Replace() { final Pattern pattern = Pattern.compile("test"); final String input = "test"; - final String replace = "$0"; + final String replace = "${0}"; final String expected = "test"; testExpression(pattern, input, replace, expected, ImmutableMap.of()); } @@ -128,7 +127,7 @@ public void testSingleMatchPartialStaticReplacePrefix() { public void testSingleMatchPartialMultipleGroupNumberReplace() { final Pattern pattern = Pattern.compile("(test)/pattern/(foo)"); final String input = "test/pattern/foo"; - final String replace = "this is a $1 pattern called $2"; + final String replace = "this is a ${1} pattern called ${2}"; final String expected = "this is a test pattern called foo"; testExpression(pattern, input, replace, expected, ImmutableMap.of()); } @@ -142,6 +141,26 @@ public void testSingleMatchPartialMultipleGroupNameReplace() { testExpression(pattern, input, replace, expected, ImmutableMap.of()); } + @Test + public void testSingleVariableReplaceAsNumber() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${1} pattern"; + final String expected = "this is a test pattern"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("1", "test")); + } + + @Test + public void testSingleVariableReplaceAsNumberPrefixed() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${prefix1:1} pattern"; + final String expected = "this is a test pattern"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("1", "test")); + testExpression(pattern, input, replace, expected, variables); + } + @Test public void testSingleMatchPartialMultipleVariableReplace() { final Pattern pattern = Pattern.compile("test/pattern/foo"); @@ -188,7 +207,9 @@ public void testMatchPrefix() { variables.put("prefix1", ImmutableMap.of("var", "var1")); variables.put("prefix2", ImmutableMap.of("var", "var2")); final RegexAndMapReplacer.Replacement replacement = testExpression(pattern, input, replace, expected, variables); - Assert.assertEquals(ImmutableList.of("prefix1:var", "prefix2:var"), replacement.getVariablesMatched()); + Assert.assertEquals( + ImmutableMap.of("prefix1", ImmutableList.of("var"), "prefix2", ImmutableList.of("var")), + replacement.getVariablesMatched()); } @Test @@ -201,7 +222,9 @@ public void testMatchPrecedence() { variables.put("prefix1", ImmutableMap.of("var", "1-var", "var2", "1-var2")); variables.put("prefix2", ImmutableMap.of("var", "2-var", "var2", "2-var2", "var3", "2-var3")); final RegexAndMapReplacer.Replacement replacement = testExpression(pattern, input, replace, expected, variables); - Assert.assertEquals(ImmutableList.of("prefix1:var2", "prefix2:var3"), replacement.getVariablesMatched()); + Assert.assertEquals( + ImmutableMap.of("prefix1", ImmutableList.of("var2"), "prefix2", ImmutableList.of("var3")), + replacement.getVariablesMatched()); } @Test From 92454bf51145f2b8943132ce7de78ed30507e58e Mon Sep 17 00:00:00 2001 From: Brandon Arp Date: Fri, 13 Apr 2018 14:58:16 -0700 Subject: [PATCH 11/11] rename injections, removals, transformations, and overwrite --- README.md | 8 +-- .../mad/sources/TransformingSource.java | 68 +++++++++---------- .../mad/sources/TransformingSourceTest.java | 16 ++--- 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/README.md b/README.md index f1facec2..e6b8f682 100644 --- a/README.md +++ b/README.md @@ -300,16 +300,16 @@ ex: name="transforming_source" transformations = [ { - inject = { + injectDimensions = { foo { value = bar - overwriteExisting = false + overwrite = false } } - remove = [ + removeDimensions = [ baz ] - findAndReplace = { + transformMetrics = { "this" = ["that"] "extract/([^/]*)/thing" = ["extract/thing/${other_dimension};my_dimension=${1}"] } diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java index 4eb21804..536c0854 100644 --- a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -134,7 +134,7 @@ public void notify(final Observable observable, final Object event) { for (final Map.Entry metric : record.getMetrics().entrySet()) { boolean found = false; final String metricName = metric.getKey(); - for (final Map.Entry> findAndReplace : transformation.getFindAndReplace().entrySet()) { + for (final Map.Entry> findAndReplace : transformation.getTransformMetrics().entrySet()) { final Pattern metricPattern = findAndReplace.getKey(); final Matcher matcher = metricPattern.matcher(metricName); if (matcher.find()) { @@ -209,11 +209,11 @@ private Key getModifiedDimensions( final Map finalTags = Maps.newHashMap(inputDimensions); // Remove the dimensions that we consumed in the replacement remove.forEach(finalTags::remove); - transformation.getRemove().forEach(finalTags::remove); - transformation.getInject().forEach( + transformation.getRemoveDimensions().forEach(finalTags::remove); + transformation.getInjectDimensions().forEach( (key, inject) -> finalTags.compute(key, (k, oldValue) -> - inject.isOverwriteExisting() || oldValue == null ? inject.getValue() : oldValue)); + inject.isOverwrite() || oldValue == null ? inject.getValue() : oldValue)); finalTags.putAll(add); return new DefaultKey(ImmutableMap.copyOf(finalTags)); @@ -299,17 +299,17 @@ public String getValue() { return _value; } - public boolean isOverwriteExisting() { - return _overwriteExisting; + public boolean isOverwrite() { + return _overwrite; } private DimensionInjection(final Builder builder) { _value = builder._value; - _overwriteExisting = builder._overwriteExisting; + _overwrite = builder._overwrite; } private final String _value; - private final boolean _overwriteExisting; + private final boolean _overwrite; /** * Implementation of the Builder pattern for {@link DimensionInjection}. @@ -341,8 +341,8 @@ public Builder setValue(final String value) { * @param value true to replace existing dimension value * @return This instance of {@link Builder}. */ - public Builder setOverwriteExisting(final Boolean value) { - _overwriteExisting = value; + public Builder setOverwrite(final Boolean value) { + _overwrite = value; return this; } @@ -350,7 +350,7 @@ public Builder setOverwriteExisting(final Boolean value) { @NotEmpty private String _value; @NotNull - private Boolean _overwriteExisting = true; + private Boolean _overwrite = true; } } @@ -360,32 +360,32 @@ public Builder setOverwriteExisting(final Boolean value) { * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) */ public static final class TransformationSet { - public ImmutableMap> getFindAndReplace() { - return _findAndReplace; + public ImmutableMap> getTransformMetrics() { + return _transformMetrics; } - public ImmutableMap getInject() { - return _inject; + public ImmutableMap getInjectDimensions() { + return _injectDimensions; } - public ImmutableList getRemove() { - return _remove; + public ImmutableList getRemoveDimensions() { + return _removeDimensions; } private TransformationSet(final Builder builder) { final ImmutableMap.Builder> findReplaceBuilder = - ImmutableMap.builderWithExpectedSize(builder._findAndReplace.size()); - for (final ImmutableMap.Entry> entry : builder._findAndReplace.entrySet()) { + ImmutableMap.builderWithExpectedSize(builder._transformMetrics.size()); + for (final ImmutableMap.Entry> entry : builder._transformMetrics.entrySet()) { findReplaceBuilder.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); } - _findAndReplace = findReplaceBuilder.build(); - _inject = builder._inject; - _remove = builder._remove; + _transformMetrics = findReplaceBuilder.build(); + _injectDimensions = builder._injectDimensions; + _removeDimensions = builder._removeDimensions; } - private final ImmutableMap _inject; - private final ImmutableList _remove; - private final ImmutableMap> _findAndReplace; + private final ImmutableMap _injectDimensions; + private final ImmutableList _removeDimensions; + private final ImmutableMap> _transformMetrics; /** * Implementation of the builder pattern for a {@link TransformationSet}. @@ -404,8 +404,8 @@ public Builder() { * @param value The find and replace expression map. * @return This instance of Builder. */ - public Builder setFindAndReplace(final ImmutableMap> value) { - _findAndReplace = value; + public Builder setTransformMetrics(final ImmutableMap> value) { + _transformMetrics = value; return this; } @@ -415,8 +415,8 @@ public Builder setFindAndReplace(final ImmutableMapBuilder. */ - public Builder setInject(final ImmutableMap value) { - _inject = value; + public Builder setInjectDimensions(final ImmutableMap value) { + _injectDimensions = value; return this; } @@ -426,17 +426,17 @@ public Builder setInject(final ImmutableMap value) { * @param value List of dimensions to inject. * @return This instance of Builder. */ - public Builder setRemove(final ImmutableList value) { - _remove = value; + public Builder setRemoveDimensions(final ImmutableList value) { + _removeDimensions = value; return this; } @NotNull - private ImmutableMap> _findAndReplace = ImmutableMap.of(); + private ImmutableMap> _transformMetrics = ImmutableMap.of(); @NotNull - private ImmutableMap _inject = ImmutableMap.of(); + private ImmutableMap _injectDimensions = ImmutableMap.of(); @NotNull - private ImmutableList _remove = ImmutableList.of(); + private ImmutableList _removeDimensions = ImmutableList.of(); } } diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java index b81e16f1..2d2a00c5 100644 --- a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -49,7 +49,7 @@ public void setUp() { _mockObserver = Mockito.mock(Observer.class); _mockSource = Mockito.mock(Source.class); _transformSetBuilder = new TransformingSource.TransformationSet.Builder() - .setFindAndReplace(ImmutableMap.of( + .setTransformMetrics(ImmutableMap.of( "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/${1}"), "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=${1}"), @@ -89,7 +89,7 @@ public void testToString() { @Test public void testMergingObserverInvalidEvent() { final TransformingSource.TransformationSet transformationSet = new TransformingSource.TransformationSet.Builder() - .setFindAndReplace(ImmutableMap.of()) + .setTransformMetrics(ImmutableMap.of()) .build(); final TransformingSource transformingSource = new TransformingSource.Builder() .setName("testMergingObserverInvalidEventTransformingSource") @@ -405,11 +405,11 @@ public void testMatchOverridesTagInCapture() { @Test public void testStaticDimensionInjection() { - _transformSetBuilder.setInject(ImmutableMap.of( + _transformSetBuilder.setInjectDimensions(ImmutableMap.of( "injected", new TransformingSource.DimensionInjection.Builder() .setValue("value") - .setOverwriteExisting(false) + .setOverwrite(false) .build())); final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( @@ -453,16 +453,16 @@ public void testStaticDimensionInjection() { @Test public void testStaticDimensionInjectionOverwrite() { - _transformSetBuilder.setInject(ImmutableMap.of( + _transformSetBuilder.setInjectDimensions(ImmutableMap.of( "injected", new TransformingSource.DimensionInjection.Builder() .setValue("new_value") - .setOverwriteExisting(true) + .setOverwrite(true) .build(), "injected_no_over", new TransformingSource.DimensionInjection.Builder() .setValue("new_value") - .setOverwriteExisting(false) + .setOverwrite(false) .build())); final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( @@ -509,7 +509,7 @@ public void testStaticDimensionInjectionOverwrite() { @Test public void testRemoveDimension() { - _transformSetBuilder.setRemove(ImmutableList.of("remove")); + _transformSetBuilder.setRemoveDimensions(ImmutableList.of("remove")); final Record matchingRecord = TestBeanFactory.createRecordBuilder() .setMetrics(ImmutableMap.of( "doesnt_match",