Telemetry processor - Allow multimatch on log body and span name (#3229)

Co-authored-by: Trask Stalnaker <[email protected]>

Author: jeanbisutti

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/ProcessorUtil.java

@@ -15,33 +15,54 @@
 
 public class ProcessorUtil {
 
+  // Rule applied on all matches for the returned string. The first match is taken to populate
+  // extracted attributes.
   public static String applyRule(
-      List<String> groupNamesList, Pattern pattern, String name, AttributesBuilder builder) {
+      List<String> groupNamesList,
+      Pattern pattern,
+      String name,
+      AttributesBuilder attributesBuilder) {
     if (groupNamesList.isEmpty()) {
       return name;
     }
     Matcher matcher = pattern.matcher(name);
-    StringBuilder sb = new StringBuilder();
+    StringBuilder output = new StringBuilder();
     int lastEnd = 0;
-    // As of now we are considering only first match.
-    if (matcher.find()) {
-      sb.append(name, lastEnd, matcher.start());
-      int innerLastEnd = matcher.start();
-      for (int i = 1; i <= groupNamesList.size(); i++) {
-        sb.append(name, innerLastEnd, matcher.start(i));
-        sb.append("{");
-        sb.append(groupNamesList.get(i - 1));
-        // add attribute key=groupNames.get(i-1), value=matcher.group(i)
-        builder.put(groupNamesList.get(i - 1), matcher.group(i));
-        sb.append("}");
-        innerLastEnd = matcher.end(i);
-      }
-      sb.append(name, innerLastEnd, matcher.end());
-      lastEnd = matcher.end();
+    boolean firstMatch = true;
+    while (matcher.find()) {
+      lastEnd =
+          applyRule(groupNamesList, name, attributesBuilder, output, lastEnd, matcher, firstMatch);
+      firstMatch = false;
     }
-    sb.append(name, lastEnd, name.length());
+    output.append(name, lastEnd, name.length());
+
+    return output.toString();
+  }
 
-    return sb.toString();
+  private static int applyRule(
+      List<String> groupNamesList,
+      String name,
+      AttributesBuilder attributesBuilder,
+      StringBuilder output,
+      int lastEnd,
+      Matcher matcher,
+      boolean firstMatch) {
+    output.append(name, lastEnd, matcher.start());
+    int innerLastEnd = matcher.start();
+    for (int i = 1; i <= groupNamesList.size(); i++) {
+      output.append(name, innerLastEnd, matcher.start(i));
+      output.append("{");
+      output.append(groupNamesList.get(i - 1));
+      // add attribute key=groupNames.get(i-1), value=matcher.group(i)
+      if (firstMatch) {
+        attributesBuilder.put(groupNamesList.get(i - 1), matcher.group(i));
+      }
+      output.append("}");
+      innerLastEnd = matcher.end(i);
+    }
+    output.append(name, innerLastEnd, matcher.end());
+    lastEnd = matcher.end();
+    return lastEnd;
   }
 
   public static List<List<String>> getGroupNamesList(List<String> toAttributeRules) {

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/processors/ExporterWithLogProcessorTest.java

@@ -227,9 +227,9 @@ void multiRuleToAttributesTest() {
     assertThat(
             Objects.requireNonNull(
                 resultA.getAttributes().get(AttributeKey.stringKey("password2"))))
-        .isEqualTo("555");
+        .isEqualTo("555"); // The first match is taken to populate the attribute
     assertThat(resultA.getBody().asString())
-        .isEqualTo("yyyPassword={password1} aba Pass={password2} xyx Pass=777 zzz");
+        .isEqualTo("yyyPassword={password1} aba Pass={password2} xyx Pass={password2} zzz");
     assertThat(
             Objects.requireNonNull(
                 resultB.getAttributes().get(AttributeKey.stringKey("password1"))))
@@ -241,6 +241,32 @@ void multiRuleToAttributesTest() {
     assertThat(resultB.getBody().asString()).isEqualTo("yyyPassword={password1} aba");
   }
 
+  @Test
+  void multiMatch() {
+    config.id = "MultiRuleToAttributes";
+    config.body = new NameConfig();
+    ToAttributeConfig toAttributeConfig = new ToAttributeConfig();
+    toAttributeConfig.rules = new ArrayList<>();
+    toAttributeConfig.rules.add("Password=(?<x>[^ ]+)");
+    config.body.toAttributes = toAttributeConfig;
+    LogRecordExporter logExporter = new ExporterWithLogProcessor(config, mockExporter);
+    TestLogRecordData mockLogA =
+        TestLogRecordData.builder()
+            .setBody("yyyPassword=123 aba Password=555 xyx")
+            .setAttributes(attributes)
+            .build();
+
+    List<LogRecordData> logs = new ArrayList<>();
+    logs.add(mockLogA);
+    logExporter.export(logs);
+
+    // verify that resulting logs are filtered in the way we want
+    List<LogRecordData> result = mockExporter.getLogs();
+    LogRecordData resultA = result.get(0);
+
+    assertThat(resultA.getBody().asString()).isEqualTo("yyyPassword={x} aba Password={x} xyx");
+  }
+
   @Test
   void simpleRenameLogTestWithLogProcessor() {
     config.id = "SimpleRenameLog";

agent/agent-tooling/src/test/java/com/microsoft/applicationinsights/agent/internal/processors/ExporterWithSpanProcessorTest.java

@@ -355,9 +355,9 @@ void multiRuleToAttributesTest() {
     assertThat(
             Objects.requireNonNull(
                 resultSpanA.getAttributes().get(AttributeKey.stringKey("password2"))))
-        .isEqualTo("555");
+        .isEqualTo("555"); // The first match is taken to populate the attribute
     assertThat(resultSpanA.getName())
-        .isEqualTo("yyyPassword={password1} aba Pass={password2} xyx Pass=777 zzz");
+        .isEqualTo("yyyPassword={password1} aba Pass={password2} xyx Pass={password2} zzz");
     assertThat(
             Objects.requireNonNull(
                 resultSpanB.getAttributes().get(AttributeKey.stringKey("password1"))))
@@ -383,7 +383,7 @@ void extractAttributesWithIncludeExcludeTest() {
     config.exclude.matchType = MatchType.STRICT;
     config.exclude.spanNames = Arrays.asList("donot/change");
     config.name.toAttributes = new ToAttributeConfig();
-    config.name.toAttributes.rules = Arrays.asList("(?<operationwebsite>.*?)$");
+    config.name.toAttributes.rules = Arrays.asList("(?<operationwebsite>.*?)/.*$");
     SpanExporter exampleExporter = new ExporterWithSpanProcessor(config, mockSpanExporter);
 
     Span spanA =
@@ -439,20 +439,20 @@ void extractAttributesWithIncludeExcludeTest() {
     SpanData resultSpanB = result.get(1);
     SpanData resultSpanC = result.get(2);
     SpanData resultSpanD = result.get(3);
-    assertThat(resultSpanA.getName()).isEqualTo("{operationwebsite}");
+    assertThat(resultSpanA.getName()).isEqualTo("{operationwebsite}/test");
     assertThat(resultSpanA.getAttributes().get(AttributeKey.stringKey("operationwebsite")))
         .isNotNull();
     assertThat(
             Objects.requireNonNull(
                 resultSpanA.getAttributes().get(AttributeKey.stringKey("operationwebsite"))))
-        .isEqualTo("svcA/test");
-    assertThat(resultSpanB.getName()).isEqualTo("{operationwebsite}");
+        .isEqualTo("svcA");
+    assertThat(resultSpanB.getName()).isEqualTo("{operationwebsite}/test");
     assertThat(resultSpanB.getAttributes().get(AttributeKey.stringKey("operationwebsite")))
         .isNotNull();
     assertThat(
             Objects.requireNonNull(
                 resultSpanB.getAttributes().get(AttributeKey.stringKey("operationwebsite"))))
-        .isEqualTo("svcB/test");
+        .isEqualTo("svcB");
     assertThat(resultSpanC.getName()).isEqualTo("svcC");
     assertThat(resultSpanD.getName()).isEqualTo("donot/change");
   }