feat(logging): Log buffering support for Logj42 and Logback

.github/pmd-ruleset.xml

@@ -350,6 +350,9 @@
     </rule>
     <rule ref="category/java/errorprone.xml/EmptyCatchBlock">
         <priority>1</priority>
+        <properties>
+            <property name="allowCommentedBlocks" value="true" />
+        </properties>
     </rule>
     <!--     <rule ref="category/java/errorprone.xml/EmptyFinalizer" /> -->
     <!--     <rule ref="category/java/errorprone.xml/EmptyFinallyBlock" /> -->
@@ -641,4 +644,4 @@
             </property>
         </properties>
     </rule>
-</ruleset>
+</ruleset>

examples/powertools-examples-core-utilities/sam/src/main/java/helloworld/App.java

@@ -97,11 +97,6 @@ public APIGatewayProxyResponseEvent handleRequest(final APIGatewayProxyRequestEv
         }
     }
 
-    @Tracing
-    private void log() {
-        log.info("inside threaded logging for function");
-    }
-
     @Tracing(namespace = "getPageContents", captureMode = CaptureMode.DISABLED)
     private String getPageContents(String address) throws IOException {
         URL url = new URL(address);

examples/powertools-examples-core-utilities/sam/src/main/java/helloworld/AppStream.java

@@ -14,46 +14,49 @@
 
 package helloworld;
 
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
-import com.fasterxml.jackson.databind.ObjectMapper;
-
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.InputStreamReader;
 import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.PrintWriter;
 import java.nio.charset.StandardCharsets;
 
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestStreamHandler;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
 import software.amazon.lambda.powertools.logging.Logging;
 import software.amazon.lambda.powertools.metrics.FlushMetrics;
 
-import java.io.InputStreamReader;
-import java.io.BufferedReader;
-import java.io.BufferedWriter;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-
 public class AppStream implements RequestStreamHandler {
     private static final ObjectMapper mapper = new ObjectMapper();
-    private final static Logger log = LogManager.getLogger(AppStream.class);
+    private static final Logger log = LoggerFactory.getLogger(AppStream.class);
 
     @Override
     @Logging(logEvent = true)
     @FlushMetrics(namespace = "ServerlessAirline", service = "payment", captureColdStart = true)
-    // RequestStreamHandler can be used instead of RequestHandler for cases when you'd like to deserialize request body or serialize response body yourself, instead of allowing that to happen automatically
-    // Note that you still need to return a proper JSON for API Gateway to handle 
-    // See Lambda Response format for examples: https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html
+    // RequestStreamHandler can be used instead of RequestHandler for cases when you'd like to deserialize request body
+    // or serialize response body yourself, instead of allowing that to happen automatically
+    // Note that you still need to return a proper JSON for API Gateway to handle
+    // See Lambda Response format for examples:
+    // https://docs.aws.amazon.com/apigateway/latest/developerguide/http-api-develop-integrations-lambda.html
     public void handleRequest(InputStream input, OutputStream output, Context context) {
         try (BufferedReader reader = new BufferedReader(new InputStreamReader(input, StandardCharsets.UTF_8));
-             PrintWriter writer = new PrintWriter(new BufferedWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8)))) {
+                PrintWriter writer = new PrintWriter(
+                        new BufferedWriter(new OutputStreamWriter(output, StandardCharsets.UTF_8)))) {
 
-            log.info("Received: " + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(mapper.readTree(reader)));
+            log.info(
+                    "Received: " + mapper.writerWithDefaultPrettyPrinter().writeValueAsString(mapper.readTree(reader)));
 
             writer.write("{\"body\": \"" + System.currentTimeMillis() + "\"} ");
         } catch (IOException e) {
             log.error("Something has gone wrong: ", e);
         }
     }
 }
-

examples/powertools-examples-core-utilities/sam/src/main/resources/log4j2.xml

@@ -4,13 +4,13 @@
         <Console name="JsonAppender" target="SYSTEM_OUT">
             <JsonTemplateLayout eventTemplateUri="classpath:LambdaJsonLayout.json" />
         </Console>
+        <BufferingAppender name="BufferedAppender" bufferAtVerbosity="DEBUG">
+            <AppenderRef ref="JsonAppender" />
+        </BufferingAppender>
     </Appenders>
     <Loggers>
-        <Logger name="JsonLogger" level="INFO" additivity="false">
-            <AppenderRef ref="JsonAppender"/>
-        </Logger>
-        <Root level="info">
-            <AppenderRef ref="JsonAppender"/>
+        <Root level="debug">
+            <AppenderRef ref="BufferedAppender" />
         </Root>
     </Loggers>
-</Configuration>
+</Configuration>

examples/powertools-examples-core-utilities/sam/template.yaml

@@ -1,4 +1,4 @@
-AWSTemplateFormatVersion: '2010-09-09'
+AWSTemplateFormatVersion: "2010-09-09"
 Transform: AWS::Serverless-2016-10-31
 Description: >
   CoreUtilities
@@ -13,7 +13,7 @@ Globals:
     Environment:
       Variables:
         # Powertools for AWS Lambda (Java) env vars: https://docs.powertools.aws.dev/lambda/java/#environment-variables
-        POWERTOOLS_LOG_LEVEL: INFO
+        POWERTOOLS_LOG_LEVEL: DEBUG # We use log buffering to buffer DEBUG logs (see log4j2.xml)
         POWERTOOLS_LOGGER_SAMPLE_RATE: 0.1
         POWERTOOLS_LOGGER_LOG_EVENT: true
         POWERTOOLS_METRICS_NAMESPACE: Coreutilities

powertools-e2e-tests/handlers/logging/pom.xml → powertools-e2e-tests/handlers/logging-log4j/pom.xml

@@ -8,20 +8,15 @@
         <version>2.3.0</version>
     </parent>
 
-    <artifactId>e2e-test-handler-logging</artifactId>
+    <artifactId>e2e-test-handler-logging-log4j</artifactId>
     <packaging>jar</packaging>
-    <name>E2E test handler – Logging</name>
+    <name>E2E test handler – Logging Log4j</name>
 
     <dependencies>
         <dependency>
             <groupId>software.amazon.lambda</groupId>
             <artifactId>powertools-logging-log4j</artifactId>
         </dependency>
-        <dependency>
-            <groupId>org.apache.logging.log4j</groupId>
-            <artifactId>log4j-layout-template-json</artifactId>
-            <version>2.25.1</version>
-        </dependency>
         <dependency>
             <groupId>org.aspectj</groupId>
             <artifactId>aspectjrt</artifactId>

powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Function.java → powertools-e2e-tests/handlers/logging-log4j/src/main/java/software/amazon/lambda/powertools/e2e/Function.java

@@ -14,12 +14,15 @@
 
 package software.amazon.lambda.powertools.e2e;
 
-import com.amazonaws.services.lambda.runtime.Context;
-import com.amazonaws.services.lambda.runtime.RequestHandler;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.MDC;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestHandler;
+
 import software.amazon.lambda.powertools.logging.Logging;
+import software.amazon.lambda.powertools.logging.PowertoolsLogging;
 
 public class Function implements RequestHandler<Input, String> {
     private static final Logger LOG = LoggerFactory.getLogger(Function.class);
@@ -29,6 +32,9 @@ public String handleRequest(Input input, Context context) {
         input.getKeys().forEach(MDC::put);
         LOG.info(input.getMessage());
 
+        // Flush buffer manually since we buffer at INFO level to test log buffering
+        PowertoolsLogging.flushBuffer();
+
         return "OK";
     }
-}
+}

powertools-e2e-tests/handlers/logging/src/main/java/software/amazon/lambda/powertools/e2e/Input.java → powertools-e2e-tests/handlers/logging-log4j/src/main/java/software/amazon/lambda/powertools/e2e/Input.java

@@ -20,9 +20,6 @@ public class Input {
     private String message;
     private Map<String, String> keys;
 
-    public Input() {
-    }
-
     public String getMessage() {
         return message;
     }

powertools-e2e-tests/handlers/logging/src/main/resources/log4j2.xml → powertools-e2e-tests/handlers/logging-log4j/src/main/resources/log4j2.xml

@@ -4,13 +4,14 @@
         <Console name="JsonAppender" target="SYSTEM_OUT">
             <JsonTemplateLayout eventTemplateUri="classpath:LambdaJsonLayout.json" />
         </Console>
+        <!-- We buffer everything to implicitly test buffer flushing -->
+        <BufferingAppender name="BufferedAppender" bufferAtVerbosity="INFO">
+            <AppenderRef ref="JsonAppender" />
+        </BufferingAppender>
     </Appenders>
     <Loggers>
         <Root level="INFO">
-            <AppenderRef ref="JsonAppender"/>
+            <AppenderRef ref="BufferedAppender" />
         </Root>
-        <Logger name="JsonLogger" level="INFO" additivity="false">
-            <AppenderRef ref="JsonAppender"/>
-        </Logger>
     </Loggers>
-</Configuration>
+</Configuration>

powertools-e2e-tests/handlers/logging-logback/pom.xml

@@ -0,0 +1,82 @@
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+    xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <parent>
+        <groupId>software.amazon.lambda</groupId>
+        <artifactId>e2e-test-handlers-parent</artifactId>
+        <version>2.3.0</version>
+    </parent>
+
+    <artifactId>e2e-test-handler-logging-logback</artifactId>
+    <packaging>jar</packaging>
+    <name>E2E test handler – Logging Logback</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>software.amazon.lambda</groupId>
+            <artifactId>powertools-logging-logback</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>org.aspectj</groupId>
+            <artifactId>aspectjrt</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-lambda-java-events</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-lambda-java-runtime-interface-client</artifactId>
+        </dependency>
+        <dependency>
+            <groupId>com.amazonaws</groupId>
+            <artifactId>aws-lambda-java-core</artifactId>
+        </dependency>
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>dev.aspectj</groupId>
+                <artifactId>aspectj-maven-plugin</artifactId>
+                <configuration>
+                    <source>${maven.compiler.source}</source>
+                    <target>${maven.compiler.target}</target>
+                    <complianceLevel>${maven.compiler.target}</complianceLevel>
+                    <aspectLibraries>
+                        <aspectLibrary>
+                            <groupId>software.amazon.lambda</groupId>
+                            <artifactId>powertools-logging</artifactId>
+                        </aspectLibrary>
+                    </aspectLibraries>
+                </configuration>
+                <executions>
+                    <execution>
+                        <goals>
+                            <goal>compile</goal>
+                        </goals>
+                    </execution>
+                </executions>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-shade-plugin</artifactId>
+            </plugin>
+        </plugins>
+    </build>
+
+    <profiles>
+        <profile>
+            <id>native-image</id>
+            <build>
+                <plugins>
+                    <plugin>
+                        <groupId>org.graalvm.buildtools</groupId>
+                        <artifactId>native-maven-plugin</artifactId>
+                    </plugin>
+                </plugins>
+            </build>
+        </profile>
+    </profiles>
+</project>

powertools-e2e-tests/handlers/logging-logback/src/main/java/software/amazon/lambda/powertools/e2e/Function.java

@@ -0,0 +1,40 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.e2e;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.slf4j.MDC;
+
+import com.amazonaws.services.lambda.runtime.Context;
+import com.amazonaws.services.lambda.runtime.RequestHandler;
+
+import software.amazon.lambda.powertools.logging.Logging;
+import software.amazon.lambda.powertools.logging.PowertoolsLogging;
+
+public class Function implements RequestHandler<Input, String> {
+    private static final Logger LOG = LoggerFactory.getLogger(Function.class);
+
+    @Logging
+    public String handleRequest(Input input, Context context) {
+        input.getKeys().forEach(MDC::put);
+        LOG.info(input.getMessage());
+
+        // Flush buffer manually since we buffer at INFO level to test log buffering
+        PowertoolsLogging.flushBuffer();
+
+        return "OK";
+    }
+}

powertools-e2e-tests/handlers/logging-logback/src/main/java/software/amazon/lambda/powertools/e2e/Input.java

@@ -0,0 +1,38 @@
+/*
+ * Copyright 2023 Amazon.com, Inc. or its affiliates.
+ * Licensed under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package software.amazon.lambda.powertools.e2e;
+
+import java.util.Map;
+
+public class Input {
+    private String message;
+    private Map<String, String> keys;
+
+    public String getMessage() {
+        return message;
+    }
+
+    public void setMessage(String message) {
+        this.message = message;
+    }
+
+    public Map<String, String> getKeys() {
+        return keys;
+    }
+
+    public void setKeys(Map<String, String> keys) {
+        this.keys = keys;
+    }
+}

powertools-e2e-tests/handlers/logging-logback/src/main/resources/META-INF/native-image/com.amazonaws/aws-lambda-java-core/reflect-config.json

@@ -0,0 +1,13 @@
+[
+  {
+    "name":"com.amazonaws.services.lambda.runtime.LambdaRuntime",
+    "methods":[{"name":"<init>","parameterTypes":[] }],
+    "fields":[{"name":"logger"}],
+    "allPublicMethods":true
+  },
+  {
+    "name":"com.amazonaws.services.lambda.runtime.LambdaRuntimeInternal",
+    "methods":[{"name":"<init>","parameterTypes":[] }],
+    "allPublicMethods":true
+  }
+]