diff --git a/README.md b/README.md index 81ca9271..e6b8f682 100644 --- a/README.md +++ b/README.md @@ -285,6 +285,80 @@ interval="10s" [[inputs.kernel]] ``` +#### TransformSource +Sometimes you'll want to transform the incoming data before aggregating. +As shown in the example collectd source, it is possible to wrap a source with +another source that will transform the data. The TransformingSource specifies +sets of transformations to apply to the input records, allowing you to add dimensions +to all metrics, remove dimensions from all metrics, and modify metrics based on their +name. + +ex: +```hocon +{ + type="com.arpnetworking.metrics.mad.sources.TransformingSource" + name="transforming_source" + transformations = [ + { + injectDimensions = { + foo { + value = bar + overwrite = false + } + } + removeDimensions = [ + baz + ] + transformMetrics = { + "this" = ["that"] + "extract/([^/]*)/thing" = ["extract/thing/${other_dimension};my_dimension=${1}"] + } + } + ] + source { + // your wrapped source goes here + } +} +``` + +tranformations is a list of TransformationSet objects. Each TransformationSet has an inject (Map\), + remove (List\) and a findAndReplace (Map\\>). + + A DimensionInjection is just a value and a boolean of whether or not to overwrite existing values. Of omitted, overwrite defaults to true. + + The keys in findAndReplace are regular expressions used to match metrics. If matched, the list of replacements is + executed, allowing for a single input metric to be recorded multiple times with different names or dimensions. + The format for the values in the list are similar to a standard java regex replacement. Variables are enclosed in ${}, + for example: + ```bash +${my_variable} +``` +Variables will be matched in the following order (first match wins): +1. capture group +2. dimension +3. environment variable + + +Variable names may also have a prefix that specifies where the value should originate. For example: +```bash +${capture:my_var} +``` +```bash +${env:MYVAR} +``` +```bash +${dimension:some_dimension} +``` + +This namespacing prevents the built-in precendence search for a matching variable. + +__Note: Any dimensions matched as part of a replacement will be removed from the resulting metric.__ + +Dimensions can also be injected into a metric. To do this, add a ';' after the replacement name of the metric +and proceed to specify key=value pairs (where both the key and value can use variables). Basically, the +replacement is processed, the string split on ';' with the first part being the metric name and anything +after the ';' being parsed as key=value pairs to be added to the metric's dimensions. + Development ----------- diff --git a/jdk-wrapper.sh b/jdk-wrapper.sh index c83f09f8..d8bf1596 100755 --- a/jdk-wrapper.sh +++ b/jdk-wrapper.sh @@ -40,6 +40,27 @@ safe_command() { fi } +checksum() { + l_file="$1" + checksum_exec="" + if command -v md5 > /dev/null; then + checksum_exec="md5" + elif command -v sha1sum > /dev/null; then + checksum_exec="sha1sum" + elif command -v shasum > /dev/null; then + checksum_exec="shasum" + fi + if [ -z "${checksum_exec}" ]; then + log_err "ERROR: No supported checksum command found!" + exit 1 + fi + cat "${l_file}" | ${checksum_exec} +} + +rand() { + awk 'BEGIN {srand();printf "%d\n", (rand() * 10^8);}' +} + download() { file="$1" if [ ! -f "${JDKW_PATH}/${file}" ]; then @@ -93,7 +114,10 @@ fi # Resolve latest version if [ "${JDKW_RELEASE}" = "latest" ]; then - JDKW_RELEASE=$(curl ${CURL_OPTIONS} -f -k -L -H 'Accept: application/json' "${JDKW_BASE_URI}/releases/latest" | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/') + latest_version_json="${TMPDIR:-/tmp}/jdkw-latest-version-$$.$(rand)" + safe_command "curl ${CURL_OPTIONS} -f -k -L -o \"${latest_version_json}\" -H 'Accept: application/json' \"${JDKW_BASE_URI}/releases/latest\"" + JDKW_RELEASE=$(cat "${latest_version_json}" | sed -e 's/.*"tag_name":"\([^"]*\)".*/\1/') + rm -f "${latest_version_json}" log_out "Resolved latest version to ${JDKW_RELEASE}" fi @@ -114,7 +138,7 @@ download "${JDKW_WRAPPER}" # Check whether this wrapper is the one specified for this version jdkw_download="${JDKW_PATH}/${JDKW_WRAPPER}" jdkw_current="$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)/$(basename "$0")" -if [ "$(cat "${jdkw_download}" | sha1sum )" != "$(cat "${jdkw_current}" | sha1sum)" ]; then +if [ "$(checksum "${jdkw_download}")" != "$(checksum "${jdkw_current}")" ]; then printf "\e[0;31m[WARNING]\e[0m Your jdk-wrapper.sh file does not match the one in your JDKW_RELEASE.\n" printf "\e[0;32mUpdate your jdk-wrapper.sh to match by running:\e[0m\n" printf "cp \"%s\" \"%s\"\n" "${jdkw_download}" "${jdkw_current}" diff --git a/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java new file mode 100644 index 00000000..536c0854 --- /dev/null +++ b/src/main/java/com/arpnetworking/metrics/mad/sources/TransformingSource.java @@ -0,0 +1,490 @@ +/** + * Copyright 2018 InscopeMetrics.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.metrics.mad.sources; + +import com.arpnetworking.commons.builder.OvalBuilder; +import com.arpnetworking.commons.builder.ThreadLocalBuilder; +import com.arpnetworking.commons.observer.Observable; +import com.arpnetworking.commons.observer.Observer; +import com.arpnetworking.logback.annotations.LogValue; +import com.arpnetworking.metrics.common.sources.BaseSource; +import com.arpnetworking.metrics.common.sources.Source; +import com.arpnetworking.metrics.mad.model.DefaultMetric; +import com.arpnetworking.metrics.mad.model.DefaultRecord; +import com.arpnetworking.metrics.mad.model.Metric; +import com.arpnetworking.metrics.mad.model.Record; +import com.arpnetworking.steno.LogValueMapFactory; +import com.arpnetworking.steno.Logger; +import com.arpnetworking.steno.LoggerFactory; +import com.arpnetworking.tsdcore.model.DefaultKey; +import com.arpnetworking.tsdcore.model.Key; +import com.arpnetworking.tsdcore.model.MetricType; +import com.arpnetworking.tsdcore.model.Quantity; +import com.arpnetworking.utility.RegexAndMapReplacer; +import com.google.common.base.MoreObjects; +import com.google.common.base.Splitter; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import net.sf.oval.constraint.NotEmpty; +import net.sf.oval.constraint.NotNull; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +/** + * Implementation of {@link Source} which wraps another {@link Source} + * and merges {@link Metric} instances within each {@link Record} + * together while, optionally, removing, injecting, and modifying dimensions + * and metrics if the name matches a regular expression with a new name generated + * through replacement of all matches in the original name. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + * @author Ville Koskela (ville dot koskela at inscopemetrics dot com) + */ +public final class TransformingSource extends BaseSource { + + @Override + public void start() { + _source.start(); + } + + @Override + public void stop() { + _source.stop(); + } + + /** + * Generate a Steno log compatible representation. + * + * @return Steno log compatible representation. + */ + @LogValue + public Object toLogValue() { + return LogValueMapFactory.builder(this) + .put("source", _source) + .put("tranformations", _transformations) + .build(); + } + + @Override + public String toString() { + return toLogValue().toString(); + } + + private TransformingSource(final Builder builder) { + super(builder); + _source = builder._source; + _transformations = builder._transformations; + + _source.attach(new TransformingObserver(this, _transformations)); + } + + private final Source _source; + private final ImmutableList _transformations; + + private static final Logger LOGGER = LoggerFactory.getLogger(TransformingSource.class); + private static final Splitter.MapSplitter TAG_SPLITTER = Splitter.on(';').omitEmptyStrings().trimResults().withKeyValueSeparator('='); + + // NOTE: Package private for testing + /* package private */ static final class TransformingObserver implements Observer { + + /* package private */ TransformingObserver( + final TransformingSource source, + final ImmutableList transformations) { + _source = source; + _transformations = transformations; + } + + @Override + public void notify(final Observable observable, final Object event) { + if (!(event instanceof Record)) { + LOGGER.error() + .setMessage("Observed unsupported event") + .addData("event", event) + .log(); + return; + } + + // Merge the metrics in the record together + final Record record = (Record) event; + final Map> mergedMetrics = Maps.newHashMap(); + final LinkedHashMap> variablesMap = new LinkedHashMap<>(); + variablesMap.put("dimension", record.getDimensions()); + variablesMap.put("env", System.getenv()); + + for (TransformationSet transformation : _transformations) { + for (final Map.Entry metric : record.getMetrics().entrySet()) { + boolean found = false; + final String metricName = metric.getKey(); + for (final Map.Entry> findAndReplace : transformation.getTransformMetrics().entrySet()) { + final Pattern metricPattern = findAndReplace.getKey(); + final Matcher matcher = metricPattern.matcher(metricName); + if (matcher.find()) { + for (final String replacement : findAndReplace.getValue()) { + final RegexAndMapReplacer.Replacement rep = + RegexAndMapReplacer.replaceAll(metricPattern, metricName, replacement, variablesMap); + final String replacedString = rep.getReplacement(); + final List consumedDimensions = rep.getVariablesMatched() + .getOrDefault("dimension", ImmutableList.of()); + + final int tagsStart = replacedString.indexOf(';'); + if (tagsStart == -1) { + // We just have a metric name. Optimize for this common case + merge( + metric.getValue(), + replacedString, + mergedMetrics, + getModifiedDimensions( + record.getDimensions(), + Collections.emptyMap(), + consumedDimensions, + transformation)); + } else { + final String newMetricName = replacedString.substring(0, tagsStart); + final Map parsedTags = TAG_SPLITTER.split(replacedString.substring(tagsStart + 1)); + merge( + metric.getValue(), + newMetricName, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), parsedTags, consumedDimensions, transformation)); + } + } + //Having "found" set here means that mapping a metric to an empty list suppresses that metric + found = true; + } + } + if (!found) { + merge( + metric.getValue(), + metricName, + mergedMetrics, + getModifiedDimensions(record.getDimensions(), Collections.emptyMap(), ImmutableList.of(), transformation)); + } + } + + // Raise the merged record event with this source's observers + // NOTE: Do not leak instances of MergingMetric since it is mutable + for (final Map.Entry> entry : mergedMetrics.entrySet()) { + _source.notify( + ThreadLocalBuilder.build( + DefaultRecord.Builder.class, + b1 -> b1.setMetrics( + entry.getValue().entrySet().stream().collect( + ImmutableMap.toImmutableMap( + Map.Entry::getKey, + e -> ThreadLocalBuilder.clone( + e.getValue(), + DefaultMetric.Builder.class)))) + .setId(record.getId()) + .setTime(record.getTime()) + .setAnnotations(record.getAnnotations()) + .setDimensions(entry.getKey().getParameters()))); + } + } + } + + private Key getModifiedDimensions( + final ImmutableMap inputDimensions, + final Map add, + final List remove, + final TransformationSet transformation) { + final Map finalTags = Maps.newHashMap(inputDimensions); + // Remove the dimensions that we consumed in the replacement + remove.forEach(finalTags::remove); + transformation.getRemoveDimensions().forEach(finalTags::remove); + transformation.getInjectDimensions().forEach( + (key, inject) -> + finalTags.compute(key, (k, oldValue) -> + inject.isOverwrite() || oldValue == null ? inject.getValue() : oldValue)); + finalTags.putAll(add); + + return new DefaultKey(ImmutableMap.copyOf(finalTags)); + } + + private void merge( + final Metric metric, + final String key, + final Map> mergedMetrics, + final Key dimensionKey) { + + final Map mergedMetricsForDimensions = + mergedMetrics.computeIfAbsent(dimensionKey, k -> Maps.newHashMap()); + final MergingMetric mergedMetric = mergedMetricsForDimensions.get(key); + if (mergedMetric == null) { + // This is the first time this metric is being merged into + mergedMetricsForDimensions.put(key, new MergingMetric(metric)); + } else if (!mergedMetric.isMergable(metric)) { + // This instance of the metric is not mergable with previous + LOGGER.error() + .setMessage("Discarding metric") + .addData("reason", "failed to merge") + .addData("metric", metric) + .addData("mergedMetric", mergedMetric) + .log(); + } else { + // Merge the new instance in + mergedMetric.merge(metric); + } + } + + private final TransformingSource _source; + private final ImmutableList _transformations; + } + + // NOTE: Package private for testing + /* package private */ static final class MergingMetric implements Metric { + + /* package private */ MergingMetric(final Metric metric) { + _type = metric.getType(); + _values.addAll(metric.getValues()); + } + + public boolean isMergable(final Metric metric) { + return _type.equals(metric.getType()); + } + + public void merge(final Metric metric) { + if (!isMergable(metric)) { + throw new IllegalArgumentException(String.format("Metric cannot be merged; metric=%s", metric)); + } + _values.addAll(metric.getValues()); + } + + @Override + public MetricType getType() { + return _type; + } + + @Override + public ImmutableList getValues() { + return _values.build(); + } + + @Override + public String toString() { + return MoreObjects.toStringHelper(this) + .add("id", Integer.toHexString(System.identityHashCode(this))) + .add("Type", _type) + .add("Values", _values) + .toString(); + } + + private final MetricType _type; + private final ImmutableList.Builder _values = ImmutableList.builder(); + } + + /** + * Represents a dimension to inject and whether or not it should overwrite the existing value (if any). + */ + public static final class DimensionInjection { + public String getValue() { + return _value; + } + + public boolean isOverwrite() { + return _overwrite; + } + + private DimensionInjection(final Builder builder) { + _value = builder._value; + _overwrite = builder._overwrite; + } + + private final String _value; + private final boolean _overwrite; + + /** + * Implementation of the Builder pattern for {@link DimensionInjection}. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ + public static final class Builder extends OvalBuilder { + /** + * Public constructor. + */ + public Builder() { + super(DimensionInjection::new); + } + + /** + * Sets the value. Required. Cannot be null. Cannot be empty. + * + * @param value The value to inject. + * @return This instance of {@link Builder}. + */ + public Builder setValue(final String value) { + _value = value; + return this; + } + + /** + * Whether to override existing dimension of this name. Optional. Cannot be null. Defaults to true. + * + * @param value true to replace existing dimension value + * @return This instance of {@link Builder}. + */ + public Builder setOverwrite(final Boolean value) { + _overwrite = value; + return this; + } + + @NotNull + @NotEmpty + private String _value; + @NotNull + private Boolean _overwrite = true; + } + } + + /** + * Represents a set of transformations to apply. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ + public static final class TransformationSet { + public ImmutableMap> getTransformMetrics() { + return _transformMetrics; + } + + public ImmutableMap getInjectDimensions() { + return _injectDimensions; + } + + public ImmutableList getRemoveDimensions() { + return _removeDimensions; + } + + private TransformationSet(final Builder builder) { + final ImmutableMap.Builder> findReplaceBuilder = + ImmutableMap.builderWithExpectedSize(builder._transformMetrics.size()); + for (final ImmutableMap.Entry> entry : builder._transformMetrics.entrySet()) { + findReplaceBuilder.put(Pattern.compile(entry.getKey()), ImmutableList.copyOf(entry.getValue())); + } + _transformMetrics = findReplaceBuilder.build(); + _injectDimensions = builder._injectDimensions; + _removeDimensions = builder._removeDimensions; + } + + private final ImmutableMap _injectDimensions; + private final ImmutableList _removeDimensions; + private final ImmutableMap> _transformMetrics; + + /** + * Implementation of the builder pattern for a {@link TransformationSet}. + */ + public static final class Builder extends OvalBuilder { + /** + * Public constructor. + */ + public Builder() { + super(TransformationSet::new); + } + + /** + * Sets find and replace expression map. Optional. Cannot be null. Defaults to empty. + * + * @param value The find and replace expression map. + * @return This instance of Builder. + */ + public Builder setTransformMetrics(final ImmutableMap> value) { + _transformMetrics = value; + return this; + } + + /** + * Sets dimensions to inject. Optional. Cannot be null. Defaults to empty. + * + * @param value List of dimensions to inject. + * @return This instance of Builder. + */ + public Builder setInjectDimensions(final ImmutableMap value) { + _injectDimensions = value; + return this; + } + + /** + * Sets dimensions to remove. Optional. Cannot be null. Defaults to empty. + * + * @param value List of dimensions to inject. + * @return This instance of Builder. + */ + public Builder setRemoveDimensions(final ImmutableList value) { + _removeDimensions = value; + return this; + } + + @NotNull + private ImmutableMap> _transformMetrics = ImmutableMap.of(); + @NotNull + private ImmutableMap _injectDimensions = ImmutableMap.of(); + @NotNull + private ImmutableList _removeDimensions = ImmutableList.of(); + } + } + + /** + * Implementation of builder pattern for {@link TransformingSource}. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ + public static final class Builder extends BaseSource.Builder { + + /** + * Public constructor. + */ + public Builder() { + super(TransformingSource::new); + } + + /** + * Sets the underlying source. Cannot be null. + * + * @param value The underlying source. + * @return This instance of {@link Builder}. + */ + public Builder setSource(final Source value) { + _source = value; + return this; + } + + /** + * Sets the transformations. Required. Cannot be null. Cannot be empty. + * + * @param value The list of transformations to apply. + * @return This instance of {@link Builder}. + */ + public Builder setTransformations(final ImmutableList value) { + _transformations = value; + return this; + } + + @Override + protected Builder self() { + return this; + } + + @NotNull + private Source _source; + @NotNull + @NotEmpty + private ImmutableList _transformations; + } +} diff --git a/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java new file mode 100644 index 00000000..9f865887 --- /dev/null +++ b/src/main/java/com/arpnetworking/utility/RegexAndMapReplacer.java @@ -0,0 +1,294 @@ +/** + * Copyright 2018 Inscope Metrics, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.utility; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * A regex replacement utility that can also replace tokens not found in the regex. + * + * ${name} in the replace string is replaced by the pattern's named capture, or by the value of the variable + * with that name from the variables map + * \ is used as an escape character and may be used to escape '$', '{', and '}' characters so that they will + * be treated as literals + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ +public final class RegexAndMapReplacer { + /** + * Replaces all instances of $n (where n is 0-9) with regex match groups, ${var} with regex capture group or variable (from map) 'var'. + * + * @param pattern pattern to use + * @param input input string to match against + * @param replace replacement string + * @param variables map of variables to include + * @return a string with replacement tokens replaced + */ + public static Replacement replaceAll( + final Pattern pattern, + final String input, + final String replace, + final LinkedHashMap> variables) { + final Matcher matcher = pattern.matcher(input); + boolean found = matcher.find(); + if (found) { + final Map> variablesUsed = Maps.newHashMap(); + final StringBuilder builder = new StringBuilder(); + int lastMatchedIndex = 0; + do { + builder.append(input.substring(lastMatchedIndex, matcher.start())); + lastMatchedIndex = matcher.end(); + appendReplacement(matcher, replace, builder, variables, variablesUsed); + found = matcher.find(); + } while (found); + // Append left-over string after the matches + if (lastMatchedIndex < input.length() - 1) { + builder.append(input.substring(lastMatchedIndex, input.length())); + } + final ImmutableMap> immutableVars = variablesUsed.entrySet() + .stream() + .collect( + Collectors.collectingAndThen( + Collectors.toMap(ImmutableMap.Entry::getKey, entry -> entry.getValue().build()), + ImmutableMap::copyOf)); + return new Replacement(builder.toString(), immutableVars); + } + return new Replacement(input, ImmutableMap.of()); + } + + private static void appendReplacement( + final Matcher matcher, + final String replacement, + final StringBuilder replacementBuilder, + final LinkedHashMap> variables, + final Map> variablesUsed) { + final StringBuilder tokenBuilder = new StringBuilder(); + int x = -1; + while (x < replacement.length() - 1) { + x++; + final char c = replacement.charAt(x); + if (c == '\\') { + x++; + processEscapedCharacter(replacement, x, replacementBuilder); + } else { + if (c == '$') { + x += writeReplacementToken(replacement, x, replacementBuilder, matcher, variables, tokenBuilder, variablesUsed); + } else { + replacementBuilder.append(c); + } + } + } + } + + private static void processEscapedCharacter(final String replacement, final int x, final StringBuilder builder) { + if (x >= replacement.length()) { + throw new IllegalArgumentException( + String.format("Improper escaping in replacement, must not have trailing '\\' at col %d: %s", x, replacement)); + } + final Character c = replacement.charAt(x); + if (c == '\\' || c == '$' || c == '{' || c == '}') { + builder.append(c); + } else { + throw new IllegalArgumentException( + String.format("Improperly escaped '%s' in replacement at col %d: %s", c, x, replacement)); + } + } + + private static int writeReplacementToken( + final String replacement, + final int offset, + final StringBuilder output, + final Matcher matcher, + final LinkedHashMap> variables, + final StringBuilder tokenBuilder, + final Map> variablesUsed) { + boolean inReplaceBrackets = false; + boolean tokenNumeric = true; + tokenBuilder.setLength(0); // reset the shared builder + int x = offset + 1; + char c = replacement.charAt(x); + + // Optionally consume the opening brace + if (c == '{') { + inReplaceBrackets = true; + x++; + c = replacement.charAt(x); + } + + if (inReplaceBrackets) { + // Consume until we hit the } + while (x < replacement.length() - 1 && c != '}') { + if (c == '\\') { + x++; + processEscapedCharacter(replacement, x, tokenBuilder); + } else { + tokenBuilder.append(c); + } + if (tokenNumeric && !Character.isDigit(c)) { + tokenNumeric = false; + } + x++; + c = replacement.charAt(x); + } + if (c != '}') { + throw new IllegalArgumentException("Invalid replacement token, expected '}' at col " + x + ": " + replacement); + } + x++; // Consume the } + output.append(getReplacement(matcher, tokenBuilder.toString(), variables, variablesUsed)); + } else { + throw new IllegalArgumentException( + String.format( + "Invalid replacement token, tokens must be surrounded by { } at col %d: %s", + x, + replacement)); + } + return x - offset - 1; + } + + private static String getReplacement( + final Matcher matcher, + final String replaceToken, + final LinkedHashMap> variables, + final Map> variablesUsed) { + final int prefixSeparatorIndex = replaceToken.indexOf(':'); + final String prefix; + final String variableName; + if (prefixSeparatorIndex == replaceToken.length() - 1) { + throw new IllegalArgumentException( + String.format("found prefix in variable replacement, but no variable name found: '%s'", + replaceToken)); + } + if (prefixSeparatorIndex != -1) { + prefix = replaceToken.substring(0, prefixSeparatorIndex); + variableName = replaceToken.substring(prefixSeparatorIndex + 1); + } else { + prefix = ""; + variableName = replaceToken; + } + if (prefix.isEmpty()) { + return replaceNonPrefix(matcher, variables, variablesUsed, variableName); + + } else { + return replacePrefix(matcher, variables, variablesUsed, prefix, variableName); + } + } + + private static String replacePrefix( + final Matcher matcher, + final LinkedHashMap> variables, + final Map> variablesUsed, + final String prefix, + final String variableName) { + if (prefix.equals("capture")) { + if (isNumeric(variableName)) { + final Integer groupIndex = Integer.valueOf(variableName); + if (groupIndex >= 0 && groupIndex <= matcher.groupCount()) { + return matcher.group(groupIndex); + } + } + + try { + return matcher.group(variableName); + } catch (final IllegalArgumentException ignored) { // No group with this name + return ""; + } + } + + // Only record variables that are not captures + variablesUsed.computeIfAbsent(prefix, key -> ImmutableList.builder()).add(variableName); + + final Map variableMap = variables.get(prefix); + if (variableMap == null) { + throw new IllegalArgumentException(String.format("could not find map for variables with prefix '%s'", prefix)); + } + return variableMap.getOrDefault(variableName, ""); + } + + private static String replaceNonPrefix( + final Matcher matcher, + final LinkedHashMap> variables, + final Map> variablesUsed, + final String variableName) { + // First try to check against the capture group number + if (isNumeric(variableName)) { + final int replaceGroup = Integer.parseInt(variableName); + if (replaceGroup >= 0 && replaceGroup <= matcher.groupCount()) { + return matcher.group(replaceGroup); + } + } + // Then try the capture group with the name + try { + return matcher.group(variableName); + } catch (final IllegalArgumentException e) { // No group with this name + // Walk through the variable maps in order to find the first match + for (final Map.Entry> entry : variables.entrySet()) { + final Map variableMap = entry.getValue(); + final String replacement = variableMap.get(variableName); + if (replacement != null) { + variablesUsed.computeIfAbsent(entry.getKey(), key -> ImmutableList.builder()).add(variableName); + return replacement; + } + } + } + return ""; + } + + private static boolean isNumeric(final String string) { + boolean isNumeric = true; + for (int x = 0; x < string.length(); x++) { + if (!Character.isDigit(string.charAt(x))) { + isNumeric = false; + break; + } + } + return isNumeric; + } + + private RegexAndMapReplacer() { } + + /** + * Describes the replacement string and variables used in it's creation. + * + * The "replacement" field is the resulting string. + * The "variablesMatched" field is a list of input variables that were matched, in prefix:variable form + */ + public static final class Replacement { + public String getReplacement() { + return _replacement; + } + + public ImmutableMap> getVariablesMatched() { + return _variablesMatched; + } + + private Replacement(final String replacement, final ImmutableMap> variablesMatched) { + + _replacement = replacement; + _variablesMatched = variablesMatched; + } + + private final String _replacement; + private final ImmutableMap> _variablesMatched; + } +} diff --git a/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java new file mode 100644 index 00000000..2d2a00c5 --- /dev/null +++ b/src/test/java/com/arpnetworking/metrics/mad/sources/TransformingSourceTest.java @@ -0,0 +1,732 @@ +/** + * Copyright 2018 InscopeMetrics.com + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.metrics.mad.sources; + +import com.arpnetworking.commons.observer.Observable; +import com.arpnetworking.commons.observer.Observer; +import com.arpnetworking.metrics.common.sources.Source; +import com.arpnetworking.metrics.mad.model.Record; +import com.arpnetworking.metrics.mad.sources.TransformingSource.MergingMetric; +import com.arpnetworking.test.TestBeanFactory; +import com.arpnetworking.test.UnorderedRecordEquality; +import com.arpnetworking.tsdcore.model.Key; +import com.arpnetworking.tsdcore.model.MetricType; +import com.arpnetworking.tsdcore.model.Quantity; +import com.arpnetworking.tsdcore.model.Unit; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Maps; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; +import org.mockito.ArgumentCaptor; +import org.mockito.Mockito; + +import java.util.Map; + +/** + * Tests for the MergingSource class. + * + * @author Ville Koskela (ville dot koskela at inscopemetrics dot com) + */ +public class TransformingSourceTest { + + @Before + public void setUp() { + _mockObserver = Mockito.mock(Observer.class); + _mockSource = Mockito.mock(Source.class); + _transformSetBuilder = new TransformingSource.TransformationSet.Builder() + .setTransformMetrics(ImmutableMap.of( + "foo/([^/]*)/bar", ImmutableList.of("foo/bar"), + "cat/([^/]*)/dog", ImmutableList.of("cat/dog", "cat/dog/${1}"), + "tagged/([^/]*)/dog", ImmutableList.of("tagged/dog;animal=${1}"), + "named/(?[^/]*)", ImmutableList.of("named/extracted_animal;extracted=${animal}"), + "tagged/([^/]*)/animal", ImmutableList.of("tagged/${animal}/animal"))); + _transformingSourceBuilder = new TransformingSource.Builder() + .setName("TransformingSourceTest") + .setTransformations(ImmutableList.of(_transformSetBuilder.build())) + .setSource(_mockSource); + } + + @Test + public void testAttach() { + _transformingSourceBuilder.build(); + Mockito.verify(_mockSource).attach(Mockito.any(Observer.class)); + } + + @Test + public void testStart() { + _transformingSourceBuilder.build().start(); + Mockito.verify(_mockSource).start(); + } + + @Test + public void testStop() { + _transformingSourceBuilder.build().stop(); + Mockito.verify(_mockSource).stop(); + } + + @Test + public void testToString() { + final String asString = _transformingSourceBuilder.build().toString(); + Assert.assertNotNull(asString); + Assert.assertFalse(asString.isEmpty()); + } + + @Test + public void testMergingObserverInvalidEvent() { + final TransformingSource.TransformationSet transformationSet = new TransformingSource.TransformationSet.Builder() + .setTransformMetrics(ImmutableMap.of()) + .build(); + final TransformingSource transformingSource = new TransformingSource.Builder() + .setName("testMergingObserverInvalidEventTransformingSource") + .setSource(_mockSource) + .setTransformations(ImmutableList.of( + transformationSet)) + .build(); + Mockito.reset(_mockSource); + new TransformingSource.TransformingObserver( + transformingSource, + ImmutableList.of(transformationSet)) + .notify(OBSERVABLE, "Not a Record"); + Mockito.verifyZeroInteractions(_mockSource); + } + + @Test(expected = IllegalArgumentException.class) + public void testMergingMetricMergeMismatchedTypes() { + final MergingMetric mergingMetric = new MergingMetric( + TestBeanFactory.createMetricBuilder() + .setType(MetricType.COUNTER) + .build()); + mergingMetric.merge(TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .build()); + } + + @Test + public void testMergeNotMatch() { + final Record nonMatchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "does_not_match", + TestBeanFactory.createMetric())) + .build(); + + final Record actualRecord = mapRecord(nonMatchingRecord); + + assertRecordsEqual(actualRecord, nonMatchingRecord); + } + + @Test + public void testMergeTwoGauges() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "foo/1/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build(), + "foo/2/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "foo/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build(), + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testDropMetricOfDifferentType() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "foo/1/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(1.23d).build())) + .build(), + "foo/2/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.TIMER) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(2.46d).build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord1 = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "foo/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(1.23d).build())) + .build())) + .build(); + final Record expectedRecord2 = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "foo/bar", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.TIMER) + .setValues(ImmutableList.of( + new Quantity.Builder().setValue(2.46d).build())) + .build())) + .build(); + + Assert.assertTrue( + String.format("expected1=%s OR expected2=%s, actual=%s", expectedRecord1, expectedRecord2, actualRecord), + UnorderedRecordEquality.equals(expectedRecord1, actualRecord) + || UnorderedRecordEquality.equals(expectedRecord2, actualRecord)); + } + + @Test + public void testReplaceWithCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "cat/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "cat/dog/sheep", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testInjectsDimension() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(matchingRecord.getMetrics()) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testExtractTagWithCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "tagged/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.builder().putAll(matchingRecord.getDimensions()).put("animal", "sheep").build()) + .setMetrics(ImmutableMap.of( + "tagged/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testInjectTagWithCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "tagged/foo/animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "animal", "frog")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.remove("animal"); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "tagged/frog/animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testMatchOverridesTagInCapture() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "named/frog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "animal", "cat")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("extracted", "frog"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "named/extracted_animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testStaticDimensionInjection() { + _transformSetBuilder.setInjectDimensions(ImmutableMap.of( + "injected", + new TransformingSource.DimensionInjection.Builder() + .setValue("value") + .setOverwrite(false) + .build())); + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("injected", "value"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testStaticDimensionInjectionOverwrite() { + _transformSetBuilder.setInjectDimensions(ImmutableMap.of( + "injected", + new TransformingSource.DimensionInjection.Builder() + .setValue("new_value") + .setOverwrite(true) + .build(), + "injected_no_over", + new TransformingSource.DimensionInjection.Builder() + .setValue("new_value") + .setOverwrite(false) + .build())); + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "injected", "old_value", + "injected_no_over", "old_value")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("injected", "new_value"); + expectedDimensions.put("injected_no_over", "old_value"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testRemoveDimension() { + _transformSetBuilder.setRemoveDimensions(ImmutableList.of("remove")); + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "remove", "_value")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.remove("remove"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "doesnt_match", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testExtractOverridesExisting() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "named/frog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .setDimensions( + ImmutableMap.of( + Key.HOST_DIMENSION_KEY, "MyHost", + Key.SERVICE_DIMENSION_KEY, "MyService", + Key.CLUSTER_DIMENSION_KEY, "MyCluster", + "extracted", "none")) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Map expectedDimensions = Maps.newHashMap(matchingRecord.getDimensions()); + expectedDimensions.put("extracted", "frog"); + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setDimensions(ImmutableMap.copyOf(expectedDimensions)) + .setMetrics(ImmutableMap.of( + "named/extracted_animal", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testReplaceWithCaptureWithTags() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "cat/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "cat/dog/sheep", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + @Test + public void testMultipleMatches() { + final Record matchingRecord = TestBeanFactory.createRecordBuilder() + .setMetrics(ImmutableMap.of( + "cat/bear/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build(), + "cat/sheep/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build())) + .build(); + + final Record actualRecord = mapRecord(matchingRecord); + + final Record expectedRecord = TestBeanFactory.createRecordBuilder() + .setAnnotations(matchingRecord.getAnnotations()) + .setTime(matchingRecord.getTime()) + .setMetrics(ImmutableMap.of( + "cat/dog", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build(), + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build(), + "cat/dog/sheep", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(2.46d) + .build())) + .build(), + "cat/dog/bear", + TestBeanFactory.createMetricBuilder() + .setType(MetricType.GAUGE) + .setValues(ImmutableList.of( + new Quantity.Builder() + .setValue(1.23d) + .setUnit(Unit.BYTE) + .build())) + .build())) + .build(); + assertRecordsEqual(actualRecord, expectedRecord); + } + + private void assertRecordsEqual(final Record actualRecord, final Record expectedRecord) { + Assert.assertTrue( + String.format("expected=%s, actual=%s", expectedRecord, actualRecord), + UnorderedRecordEquality.equals(expectedRecord, actualRecord)); + } + + private Record mapRecord(final Record record) { + _transformingSourceBuilder.setTransformations(ImmutableList.of(_transformSetBuilder.build())); + final Source transformingSource = _transformingSourceBuilder.build(); + transformingSource.attach(_mockObserver); + notify(_mockSource, record); + + final ArgumentCaptor argument = ArgumentCaptor.forClass(Record.class); + Mockito.verify(_mockObserver).notify(Mockito.same(transformingSource), argument.capture()); + return argument.getValue(); + } + + private static void notify(final Observable observable, final Object event) { + final ArgumentCaptor argument = ArgumentCaptor.forClass(Observer.class); + Mockito.verify(observable).attach(argument.capture()); + for (final Observer observer : argument.getAllValues()) { + observer.notify(observable, event); + } + } + + private Observer _mockObserver; + private Source _mockSource; + private TransformingSource.Builder _transformingSourceBuilder; + private TransformingSource.TransformationSet.Builder _transformSetBuilder; + + private static final Observable OBSERVABLE = new Observable() { + @Override + public void attach(final Observer observer) { + } + + @Override + public void detach(final Observer observer) { + } + }; +} diff --git a/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java b/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java index 04d1e446..d5906965 100644 --- a/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java +++ b/src/test/java/com/arpnetworking/test/UnorderedRecordEquality.java @@ -42,7 +42,8 @@ public final class UnorderedRecordEquality { */ public static boolean equals(final Record r1, final Record r2) { if (!r1.getTime().equals(r2.getTime()) - || !r1.getAnnotations().equals(r2.getAnnotations())) { + || !r1.getAnnotations().equals(r2.getAnnotations()) + || !r1.getDimensions().equals(r2.getDimensions())) { return false; } diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java new file mode 100644 index 00000000..969e2bee --- /dev/null +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapBenchmarkTest.java @@ -0,0 +1,51 @@ +/** + * Copyright 2018 Inscope Metrics, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.utility; + +import com.carrotsearch.junitbenchmarks.BenchmarkOptions; +import com.carrotsearch.junitbenchmarks.BenchmarkRule; +import org.junit.Rule; +import org.junit.Test; +import org.junit.rules.TestRule; + +import java.util.LinkedHashMap; +import java.util.regex.Pattern; + +/** + * Benchmark tests comparing the RegexAndMapReplacer class against the built-in regex replacement. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ +public final class RegexAndMapBenchmarkTest { + + @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) + @Test + public void testRegexAndMap() { + final String result = RegexAndMapReplacer.replaceAll(PATTERN, INPUT, REPLACE, new LinkedHashMap<>()).getReplacement(); + } + + @BenchmarkOptions(benchmarkRounds = 2000000, warmupRounds = 50000) + @Test + public void testRegex() { + final String result = PATTERN.matcher(INPUT).replaceAll(REPLACE); + } + + @Rule + public TestRule _benchmarkRun = new BenchmarkRule(); + private static final String REPLACE = "this is a ${g1} pattern called ${g2}"; + private static final Pattern PATTERN = Pattern.compile("(?test)/pattern/(?foo)"); + private static final String INPUT = "test/pattern/foo"; +} diff --git a/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java new file mode 100644 index 00000000..450967b2 --- /dev/null +++ b/src/test/java/com/arpnetworking/utility/RegexAndMapReplacerTest.java @@ -0,0 +1,296 @@ +/** + * Copyright 2018 Inscope Metrics, Inc + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.arpnetworking.utility; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import org.junit.Assert; +import org.junit.Test; + +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.regex.Pattern; + +/** + * Tests for the {@link RegexAndMapReplacer} class. + * + * @author Brandon Arp (brandon dot arp at inscopemetrics dot com) + */ +public final class RegexAndMapReplacerTest { + @Test + public void testNoMatch() { + final Pattern pattern = Pattern.compile("test"); + final String input = "wont match"; + final String replace = "$0"; + final String expected = "wont match"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidEscape() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${\\avariable}"; // \a is an invalid escape sequence + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test(expected = IllegalArgumentException.class) + public void testMissingClosingCurly() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${0"; // no ending } + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidEscapeAtEnd() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${0}\\"; // trailing \ + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidNoVariableName() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${prefix:}"; // variable prefix, but no name + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidReplacementTokenMissingOpen() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "$variable"; // replacement variable has no { + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test(expected = IllegalArgumentException.class) + public void testInvalidReplacementTokenNumeric() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "$0"; // replacement variable has no { + final String result = RegexAndMapReplacer.replaceAll(pattern, input, replace, new LinkedHashMap<>()).getReplacement(); + } + + @Test + public void testGroup0Replace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "${0}"; + final String expected = "test"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchFullStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test"; + final String replace = "replace"; + final String expected = "replace"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test string"; + final String replace = "replace"; + final String expected = "replace string"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialStaticReplacePrefix() { + final Pattern pattern = Pattern.compile("test"); + final String input = "some test string"; + final String replace = "replace"; + final String expected = "some replace string"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialMultipleGroupNumberReplace() { + final Pattern pattern = Pattern.compile("(test)/pattern/(foo)"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${1} pattern called ${2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleMatchPartialMultipleGroupNameReplace() { + final Pattern pattern = Pattern.compile("(?test)/pattern/(?foo)"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testSingleVariableReplaceAsNumber() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${1} pattern"; + final String expected = "this is a test pattern"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("1", "test")); + } + + @Test + public void testSingleVariableReplaceAsNumberPrefixed() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${prefix1:1} pattern"; + final String expected = "this is a test pattern"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("1", "test")); + testExpression(pattern, input, replace, expected, variables); + } + + @Test + public void testSingleMatchPartialMultipleVariableReplace() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "test", "g2", "foo")); + } + + @Test + public void testSingleMatchPartialMultipleVariableWithEscapeReplace() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern \\\\called\\\\ ${g2}"; + final String expected = "this is a test pattern \\called\\ foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "test", "g2", "foo")); + } + + @Test + public void testSingleMatchPartialMultipleVariableWithEscapeTokenReplace() { + final Pattern pattern = Pattern.compile("test/pattern/foo"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${\\\\g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("\\g1", "test", "g2", "foo")); + } + + @Test + public void testSingleMatchPartialMultipleGroupNameOverridesVariablesReplace() { + final Pattern pattern = Pattern.compile("(?test)/pattern/(?foo)"); + final String input = "test/pattern/foo"; + final String replace = "this is a ${g1} pattern called ${g2}"; + final String expected = "this is a test pattern called foo"; + testExpression(pattern, input, replace, expected, ImmutableMap.of("g1", "bad", "g2", "value")); + } + + @Test + public void testMatchPrefix() { + final Pattern pattern = Pattern.compile("input_string_(?.*)_(.*)"); + final String input = "input_string_ending_ending2"; + final String replace = "${prefix1:var}_${prefix2:var}_${capture:var}_${capture:2}"; + final String expected = "var1_var2_ending_ending2"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "var1")); + variables.put("prefix2", ImmutableMap.of("var", "var2")); + final RegexAndMapReplacer.Replacement replacement = testExpression(pattern, input, replace, expected, variables); + Assert.assertEquals( + ImmutableMap.of("prefix1", ImmutableList.of("var"), "prefix2", ImmutableList.of("var")), + replacement.getVariablesMatched()); + } + + @Test + public void testMatchPrecedence() { + final Pattern pattern = Pattern.compile("input_string_(?.*)"); + final String input = "input_string_ending"; + final String replace = "${var}_${var2}_${var3}"; + final String expected = "ending_1-var2_2-var3"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "1-var", "var2", "1-var2")); + variables.put("prefix2", ImmutableMap.of("var", "2-var", "var2", "2-var2", "var3", "2-var3")); + final RegexAndMapReplacer.Replacement replacement = testExpression(pattern, input, replace, expected, variables); + Assert.assertEquals( + ImmutableMap.of("prefix1", ImmutableList.of("var2"), "prefix2", ImmutableList.of("var3")), + replacement.getVariablesMatched()); + } + + @Test + public void testMatchMissingVars() { + final Pattern pattern = Pattern.compile("input_string"); + final String input = "input_string_ending"; + final String replace = "${prefix1:missing}1${capture:missing}2${missing}"; + final String expected = "12_ending"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "var1")); + testExpression(pattern, input, replace, expected, variables); + } + + @Test(expected = IllegalArgumentException.class) + public void testMatchInvalidPrefix() { + final Pattern pattern = Pattern.compile("input_string_(?.*)"); + final String input = "input_string_ending"; + final String replace = "${notfound:var}_value"; + final String expected = "illegal arg exception"; + final LinkedHashMap> variables = new LinkedHashMap<>(); + variables.put("prefix1", ImmutableMap.of("var", "var1")); + testExpression(pattern, input, replace, expected, variables); + } + + @Test + public void testMultipleMatchFullStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "testtest"; + final String replace = "replace"; + final String expected = "replacereplace"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + @Test + public void testMultipleMatchPartialStaticReplace() { + final Pattern pattern = Pattern.compile("test"); + final String input = "test string test"; + final String replace = "replace"; + final String expected = "replace string replace"; + testExpression(pattern, input, replace, expected, ImmutableMap.of()); + } + + private RegexAndMapReplacer.Replacement testExpression( + final Pattern pattern, + final String input, + final String replace, + final String expected, + final ImmutableMap variables) { + final LinkedHashMap> variablesMap = new LinkedHashMap<>(); + variablesMap.put("myvars", variables); + return testExpression(pattern, input, replace, expected, variablesMap); + } + + private RegexAndMapReplacer.Replacement testExpression( + final Pattern pattern, + final String input, + final String replace, + final String expected, + final LinkedHashMap> variables) { + final RegexAndMapReplacer.Replacement replacement = RegexAndMapReplacer.replaceAll(pattern, input, replace, variables); + final String result = replacement.getReplacement(); + Assert.assertEquals(expected, result); + try { + final String stockResult = pattern.matcher(input).replaceAll(replace); + Assert.assertEquals(expected, stockResult); + } catch (final IllegalArgumentException ignored) { } + return replacement; + } +}