add logprocessor to agent (#1713)

* add logprocessor to agent

* comments addressed, added smoketest for log processor, fixed bug in telemetryprocessorOpenTelemetryConfigurer

* StringBuffer -> StringBuilder

* addressing comments

* removed lognames, fixed smoke test

* replace name with body in log processor, address comments

Co-authored-by: Trask Stalnaker <[email protected]>

Author: kryalama

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/AgentProcessor.java

@@ -1,19 +1,40 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
 package com.microsoft.applicationinsights.agent.internal.processors;
 
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.MatchType;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorAttribute;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorIncludeExclude;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import org.checkerframework.checker.nullness.qual.Nullable;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Map.Entry;
 import java.util.regex.Pattern;
 
-import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorAttribute;
-import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorIncludeExclude;
-import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.MatchType;
-import io.opentelemetry.api.common.AttributeKey;
-import io.opentelemetry.sdk.trace.data.SpanData;
-import org.checkerframework.checker.nullness.qual.Nullable;
-
 public abstract class AgentProcessor {
     private final @Nullable IncludeExclude include;
     private final @Nullable IncludeExclude exclude;
@@ -36,13 +57,12 @@ protected static AttributeProcessor.IncludeExclude getNormalizedIncludeExclude(P
         return exclude;
     }
 
+
     public static abstract class IncludeExclude {
         // Function to compare span with user provided span names or span patterns
-        public abstract boolean isMatch(SpanData span);
-
+        public abstract boolean isMatch(SpanData span, boolean isLog);
     }
 
-    // ok to have this class cover both spanNames and logNames
     public static class StrictIncludeExclude extends IncludeExclude {
         private final List<ProcessorAttribute> attributes;
         private final List<String> spanNames;
@@ -65,10 +85,12 @@ public static StrictIncludeExclude create(ProcessorIncludeExclude includeExclude
         }
 
         // Function to compare span with user provided span names
-        public boolean isMatch(SpanData span) {
-            if (!spanNames.isEmpty() && !spanNames.contains(span.getName())) {
-                // span name doesn't match
-                return false;
+        public boolean isMatch(SpanData span, boolean isLog) {
+            if (isLog) {
+                // If user provided spanNames , then donot include log in the include/exclude criteria
+                if(!spanNames.isEmpty()) return false;
+            } else {
+                if (!spanNames.isEmpty() && !spanNames.contains(span.getName())) return false;
             }
             return this.checkAttributes(span);
         }
@@ -115,13 +137,13 @@ public static RegexpIncludeExclude create(ProcessorIncludeExclude includeExclude
                     }
                 }
             }
+
             List<Pattern> spanPatterns = new ArrayList<>();
             if (includeExclude.spanNames != null) {
                 for (String regex : includeExclude.spanNames) {
                     spanPatterns.add(Pattern.compile(regex));
                 }
             }
-
             return new RegexpIncludeExclude(spanPatterns, attributeKeyValuePatterns);
         }
 
@@ -130,8 +152,8 @@ private static boolean isAttributeValueMatch(String attributeValue, Pattern valu
             return valuePattern.matcher(attributeValue).find();
         }
 
-        private boolean isPatternFound(SpanData span) {
-            for (Pattern pattern : spanPatterns) {
+        private static boolean isPatternFound(SpanData span, List<Pattern> patterns) {
+            for (Pattern pattern : patterns) {
                 if (pattern.matcher(span.getName()).find()) {
                     // pattern matches the span!!!
                     return true;
@@ -141,10 +163,13 @@ private boolean isPatternFound(SpanData span) {
             return false;
         }
 
-        // Function to compare span with user provided span patterns
-        public boolean isMatch(SpanData span) {
-            if (!spanPatterns.isEmpty() && !isPatternFound(span)) {
-                return false;
+        // Function to compare span/log with user provided span patterns/log patterns
+        public boolean isMatch(SpanData span, boolean isLog) {
+            if (isLog) {
+                // If user provided spanNames, then do not include log in the include/exclude criteria
+                if (!spanPatterns.isEmpty()) return false;
+            } else {
+                if (!spanPatterns.isEmpty() && !isPatternFound(span, spanPatterns)) return false;
             }
             return checkAttributes(span);
         }

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/AttributeProcessor.java

@@ -1,7 +1,25 @@
-package com.microsoft.applicationinsights.agent.internal.processors;
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
-import java.util.List;
-import java.util.regex.Matcher;
+package com.microsoft.applicationinsights.agent.internal.processors;
 
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorAction;
 import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorConfig;
@@ -12,6 +30,9 @@
 import org.apache.commons.codec.digest.DigestUtils;
 import org.checkerframework.checker.nullness.qual.Nullable;
 
+import java.util.List;
+import java.util.regex.Matcher;
+
 // structure which only allows valid data
 // normalization has to occur before construction
 public class AttributeProcessor extends AgentProcessor {
@@ -33,6 +54,18 @@ public static AttributeProcessor create(ProcessorConfig config) {
         return new AttributeProcessor(config.actions, normalizedInclude, normalizedExclude);
     }
 
+    // this won't be needed once we update to 0.13.0
+    // see https://github.com/open-telemetry/opentelemetry-java/pull/2284
+    public static String getAttribute(Attributes attributes, AttributeKey<String> key) {
+        Object existingValueObj = attributes.get(key);
+        // checking the return type won't be needed once we update to 0.13.0
+        // see https://github.com/open-telemetry/opentelemetry-java/pull/2284
+        if (existingValueObj instanceof String) {
+            return (String) existingValueObj;
+        }
+        return null;
+    }
+
     // Function to process actions
     public SpanData processActions(SpanData span) {
         SpanData updatedSpan = span;
@@ -149,19 +182,6 @@ private SpanData processExtractAction(SpanData span, ProcessorAction actionObj)
         return new MySpanData(span, builder.build());
     }
 
-    // this won't be needed once we update to 0.13.0
-    // see https://github.com/open-telemetry/opentelemetry-java/pull/2284
-    public static String getAttribute(Attributes attributes, AttributeKey<String> key) {
-        Object existingValueObj = attributes.get(key);
-        // checking the return type won't be needed once we update to 0.13.0
-        // see https://github.com/open-telemetry/opentelemetry-java/pull/2284
-        if (existingValueObj instanceof String) {
-            return (String) existingValueObj;
-        } else {
-            return null;
-        }
-    }
-
     @SuppressWarnings("unchecked")
     private void putIntoBuilder(AttributesBuilder builder, AttributeKey<?> key, Object value) {
         switch (key.getType()) {

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/ExporterWithAttributeProcessor.java

@@ -1,17 +1,37 @@
-package com.microsoft.applicationinsights.agent.internal.processors;
-
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.List;
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
 
+package com.microsoft.applicationinsights.agent.internal.processors;
 
-import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorConfig;
 import com.microsoft.applicationinsights.agent.internal.processors.AgentProcessor.IncludeExclude;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorConfig;
 import com.microsoft.applicationinsights.customExceptions.FriendlyException;
 import io.opentelemetry.sdk.common.CompletableResultCode;
 import io.opentelemetry.sdk.trace.data.SpanData;
 import io.opentelemetry.sdk.trace.export.SpanExporter;
 
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
 public class ExporterWithAttributeProcessor implements SpanExporter {
 
     private final SpanExporter delegate;
@@ -27,21 +47,23 @@ public ExporterWithAttributeProcessor(ProcessorConfig config, SpanExporter deleg
     @Override
     public CompletableResultCode export(Collection<SpanData> spans) {
         // we need to filter attributes before passing on to delegate
-            List<SpanData> copy = new ArrayList<>();
-            for (SpanData span : spans) {
-                copy.add(process(span));
-            }
-            return delegate.export(copy);
+        List<SpanData> copy = new ArrayList<>();
+        for (SpanData span : spans) {
+            copy.add(process(span));
+        }
+        return delegate.export(copy);
     }
 
     private SpanData process(SpanData span) {
         IncludeExclude include = attributeProcessor.getInclude();
-        if (include != null && !include.isMatch(span)) {
-            //If Not included we can skip further processing
+        boolean isLog = ProcessorUtil.isSpanOfTypeLog(span);
+        if (include != null && !include.isMatch(span, isLog)) {
+            //If not included we can skip further processing
             return span;
         }
         IncludeExclude exclude = attributeProcessor.getExclude();
-        if (exclude != null && exclude.isMatch(span)) {
+        if (exclude != null && exclude.isMatch(span, isLog)) {
+            //If excluded we can skip further processing
             return span;
         }
         return attributeProcessor.processActions(span);

agent/agent-tooling/src/main/java/com/microsoft/applicationinsights/agent/internal/processors/ExporterWithLogProcessor.java

@@ -0,0 +1,84 @@
+/*
+ * ApplicationInsights-Java
+ * Copyright (c) Microsoft Corporation
+ * All rights reserved.
+ *
+ * MIT License
+ * Permission is hereby granted, free of charge, to any person obtaining a copy of this
+ * software and associated documentation files (the ""Software""), to deal in the Software
+ * without restriction, including without limitation the rights to use, copy, modify, merge,
+ * publish, distribute, sublicense, and/or sell copies of the Software, and to permit
+ * persons to whom the Software is furnished to do so, subject to the following conditions:
+ * The above copyright notice and this permission notice shall be included in all copies or
+ * substantial portions of the Software.
+ * THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+ * INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+ * PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+ * FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+ * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+ * DEALINGS IN THE SOFTWARE.
+ */
+
+package com.microsoft.applicationinsights.agent.internal.processors;
+
+import com.microsoft.applicationinsights.agent.internal.processors.AgentProcessor.IncludeExclude;
+import com.microsoft.applicationinsights.agent.internal.wasbootstrap.configuration.Configuration.ProcessorConfig;
+import com.microsoft.applicationinsights.customExceptions.FriendlyException;
+import io.opentelemetry.sdk.common.CompletableResultCode;
+import io.opentelemetry.sdk.trace.data.SpanData;
+import io.opentelemetry.sdk.trace.export.SpanExporter;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+public class ExporterWithLogProcessor implements SpanExporter {
+
+    private final SpanExporter delegate;
+    private final LogProcessor logProcessor;
+
+    // caller should check config.isValid before creating
+    public ExporterWithLogProcessor(ProcessorConfig config, SpanExporter delegate) throws FriendlyException {
+        config.validate();
+        logProcessor = LogProcessor.create(config);
+        this.delegate = delegate;
+    }
+
+    @Override
+    public CompletableResultCode export(Collection<SpanData> spans) {
+        // we need to filter attributes before passing on to delegate
+        List<SpanData> copy = new ArrayList<>();
+        for (SpanData span : spans) {
+            copy.add(process(span));
+        }
+        return delegate.export(copy);
+    }
+
+    private SpanData process(SpanData span) {
+        if (!ProcessorUtil.isSpanOfTypeLog(span)) {
+            return span;
+        }
+        IncludeExclude include = logProcessor.getInclude();
+        if (include != null && !include.isMatch(span, true)) {
+            //If Not included we can skip further processing
+            return span;
+        }
+        IncludeExclude exclude = logProcessor.getExclude();
+        if (exclude != null && exclude.isMatch(span, true)) {
+            return span;
+        }
+
+        SpanData updatedSpan = logProcessor.processFromAttributes(span);
+        return logProcessor.processToAttributes(updatedSpan);
+    }
+
+    @Override
+    public CompletableResultCode flush() {
+        return delegate.flush();
+    }
+
+    @Override
+    public CompletableResultCode shutdown() {
+        return delegate.shutdown();
+    }
+}