ECS formatter for JUL (java.util.logging) (#69)

Author: redrathnure

README.md

@@ -112,6 +112,7 @@ For nested structures consider prefixing with `custom.` to make sure you won't g
 - [Logback](logback-ecs-encoder/README.md) (default for Spring Boot)
 - [Log4j2](log4j2-ecs-layout/README.md)
 - [Log4j](log4j-ecs-layout/README.md)
+- [java.util.logging](jul-ecs-formatter/README.md)
 
 ### Step 2: Enable APM log correlation (optional)
 If you are using the Elastic APM Java agent,

ecs-logging-core/src/main/java/co/elastic/logging/EcsJsonSerializer.java

@@ -52,9 +52,11 @@ public static void serializeObjectEnd(StringBuilder builder) {
     }
 
     public static void serializeLoggerName(StringBuilder builder, String loggerName) {
-        builder.append("\"log.logger\":\"");
-        JsonUtils.quoteAsString(loggerName, builder);
-        builder.append("\",");
+        if (loggerName != null) {
+            builder.append("\"log.logger\":\"");
+            JsonUtils.quoteAsString(loggerName, builder);
+            builder.append("\",");
+        }
     }
 
     public static void serializeThreadName(StringBuilder builder, String threadName) {
@@ -65,6 +67,11 @@ public static void serializeThreadName(StringBuilder builder, String threadName)
         }
     }
 
+    public static void serializeThreadId(StringBuilder builder, long threadId) {
+        builder.append("\"process.thread.id\":");
+        builder.append(threadId);
+        builder.append(",");
+    }
     public static void serializeFormattedMessage(StringBuilder builder, String message) {
         builder.append("\"message\":\"");
         JsonUtils.quoteAsString(message, builder);
@@ -134,9 +141,12 @@ public static void serializeOrigin(StringBuilder builder, String fileName, Strin
         builder.append("\",");
         builder.append("\"function\":\"");
         JsonUtils.quoteAsString(methodName, builder);
-        builder.append("\",");
-        builder.append("\"file.line\":");
-        builder.append(lineNumber);
+        builder.append('"');
+        if (lineNumber >= 0) {
+            builder.append(',');
+            builder.append("\"file.line\":");
+            builder.append(lineNumber);
+        }
         builder.append("},");
     }
 

ecs-logging-core/src/test/java/co/elastic/logging/AbstractEcsLoggingTest.java

@@ -61,9 +61,10 @@ void testSimpleLog() throws Exception {
 
     @Test
     void testThreadContext() throws Exception {
-        putMdc("foo", "bar");
-        debug("test");
-        assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
+        if (putMdc("foo", "bar")) {
+            debug("test");
+            assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
+        }
     }
 
     @Test
@@ -76,14 +77,16 @@ void testThreadContextStack() throws Exception {
 
     @Test
     void testMdc() throws Exception {
-        putMdc("transaction.id", "0af7651916cd43dd8448eb211c80319c");
-        putMdc("span.id", "foo");
-        putMdc("foo", "bar");
-        debug("test");
-        assertThat(getLastLogLine().get("labels.transaction.id")).isNull();
-        assertThat(getLastLogLine().get("transaction.id").textValue()).isEqualTo("0af7651916cd43dd8448eb211c80319c");
-        assertThat(getLastLogLine().get("span.id").textValue()).isEqualTo("foo");
-        assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
+        if (putMdc("transaction.id", "0af7651916cd43dd8448eb211c80319c")) {
+            putMdc("span.id", "foo");
+            putMdc("foo", "bar");
+            debug("test");
+            assertThat(getLastLogLine().get("labels.transaction.id")).isNull();
+            assertThat(getLastLogLine().get("transaction.id").textValue())
+                    .isEqualTo("0af7651916cd43dd8448eb211c80319c");
+            assertThat(getLastLogLine().get("span.id").textValue()).isEqualTo("foo");
+            assertThat(getLastLogLine().get("foo").textValue()).isEqualTo("bar");
+        }
     }
 
     @Test
@@ -113,7 +116,9 @@ void testLogOrigin() throws Exception {
         assertThat(getLastLogLine().get("log.origin").get("file.line").intValue()).isPositive();
     }
 
-    public abstract void putMdc(String key, String value);
+    public boolean putMdc(String key, String value) {
+        return false;
+    }
 
     public boolean putNdc(String message) {
         return false;

jul-ecs-formatter/README.md

@@ -0,0 +1,41 @@
+# ECS formatter for JUL
+
+Formatter for JUL (java.util.logging) which produce ECS-compatible records. May be useful for applications which use JUL as primary logging framework (e.g. Apache Tomcat).
+
+## Step 1: add dependency
+
+Latest version: [![Maven Central](https://img.shields.io/maven-central/v/co.elastic.logging/jul-ecs-formatter.svg)](https://search.maven.org/search?q=g:co.elastic.logging%20AND%20a:jul-ecs-formatter)
+
+Add a dependency to your application
+```xml
+<dependency>
+    <groupId>co.elastic.logging</groupId>
+    <artifactId>jul-ecs-formatter</artifactId>
+    <version>${ecs-logging-java.version}</version>
+</dependency>
+```
+If you are not using a dependency management tool, like maven, you have to add both, `jul-ecs-formatter` and `ecs-logging-core` jars manually to the classpath. For example to the `$CATALINA_HOME/lib` directory.
+
+## Step 2: use the `EcsFormatter`
+
+Specify `co.elastic.logging.jul.EcsFormatter` as `formatter` for the required log handler. 
+
+## Example
+For example `$CATALINA_HOME/conf/logging.properties`
+
+```properties
+java.util.logging.ConsoleHandler.level = FINE
+java.util.logging.ConsoleHandler.formatter = co.elastic.logging.jul.EcsFormatter
+co.elastic.logging.jul.EcsFormatter.serviceName=my-app
+```
+
+## Layout Parameters
+
+|Parameter name   |Type   |Default|Description|
+|-----------------|-------|-------|-----------|
+|serviceName      |String |       |Sets the `service.name` field so you can filter your logs by a particular service |
+|eventDataset     |String |`${serviceName}.log`|Sets the `event.dataset` field used by the machine learning job of the Logs app to look for anomalies in the log rate. |
+|stackTraceAsArray|boolean|`false`|Serializes the [`error.stack_trace`](https://www.elastic.co/guide/en/ecs/current/ecs-error.html) as a JSON array where each element is in a new line to improve readability. Note that this requires a slightly more complex [Filebeat configuration](../README.md#when-stacktraceasarray-is-enabled).|
+|includeOrigin    |boolean|`false`|If `true`, adds the [`log.origin.file.name`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html), [`log.origin.file.line`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) and [`log.origin.function`](https://www.elastic.co/guide/en/ecs/current/ecs-log.html) fields. Note that JUL does not stores line number and `log.origin.file.line` will have '1' value. |
+
+ 

jul-ecs-formatter/pom.xml

@@ -0,0 +1,32 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <parent>
+        <artifactId>ecs-logging-java-parent</artifactId>
+        <groupId>co.elastic.logging</groupId>
+        <version>0.3.1-SNAPSHOT</version>
+    </parent>
+    <modelVersion>4.0.0</modelVersion>
+
+    <properties>
+        <parent.base.dir>${project.basedir}/..</parent.base.dir>
+    </properties>
+
+    <artifactId>jul-ecs-formatter</artifactId>
+    <name>${project.groupId}:${project.artifactId}</name>
+
+    <dependencies>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>ecs-logging-core</artifactId>
+            <version>${project.version}</version>
+        </dependency>
+        <dependency>
+            <groupId>${project.groupId}</groupId>
+            <artifactId>ecs-logging-core</artifactId>
+            <version>${project.version}</version>
+            <type>test-jar</type>
+            <scope>test</scope>
+        </dependency>
+    </dependencies>
+
+</project>

jul-ecs-formatter/src/main/java/co/elastic/logging/jul/EcsFormatter.java

@@ -0,0 +1,116 @@
+/*-
+ * #%L
+ * Java ECS logging
+ * %%
+ * Copyright (C) 2019 - 2020 Elastic and contributors
+ * %%
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * 
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ * 
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ * #L%
+ */
+package co.elastic.logging.jul;
+
+import java.util.logging.Formatter;
+import java.util.logging.LogManager;
+import java.util.logging.LogRecord;
+
+import co.elastic.logging.EcsJsonSerializer;
+
+public class EcsFormatter extends Formatter {
+
+    private static final String UNKNOWN_FILE = "<Unknown>";
+    
+    private boolean stackTraceAsArray;
+    private String serviceName;
+    private boolean includeOrigin;
+    private String eventDataset;
+
+    /**
+     * Default constructor. Will read configuration from LogManager properties.
+     */
+    public EcsFormatter() {
+        serviceName = getProperty("co.elastic.logging.jul.EcsFormatter.serviceName", null);
+        includeOrigin = Boolean.getBoolean(getProperty("co.elastic.logging.jul.EcsFormatter.includeOrigin", "false"));
+        stackTraceAsArray = Boolean
+                .getBoolean(getProperty("co.elastic.logging.jul.EcsFormatter.stackTraceAsArray", "false"));
+        eventDataset = getProperty("co.elastic.logging.jul.EcsFormatter.eventDataset", null);
+        eventDataset = EcsJsonSerializer.computeEventDataset(eventDataset, serviceName);
+    }
+
+    @Override
+    public String format(final LogRecord record) {
+        final StringBuilder builder = new StringBuilder();
+        EcsJsonSerializer.serializeObjectStart(builder, record.getMillis());
+        EcsJsonSerializer.serializeLogLevel(builder, record.getLevel().getName());
+        EcsJsonSerializer.serializeFormattedMessage(builder, record.getMessage());
+        EcsJsonSerializer.serializeServiceName(builder, serviceName);
+        EcsJsonSerializer.serializeEventDataset(builder, eventDataset);
+        EcsJsonSerializer.serializeThreadId(builder, record.getThreadID());
+        EcsJsonSerializer.serializeLoggerName(builder, record.getLoggerName());
+        if (includeOrigin && record.getSourceClassName() != null && record.getSourceMethodName() != null) {
+            EcsJsonSerializer.serializeOrigin(builder, buildFileName(record.getSourceClassName()), record.getSourceMethodName(), -1);
+        }
+        final Throwable throwableInformation = record.getThrown();
+        if (throwableInformation != null) {
+            EcsJsonSerializer.serializeException(builder, throwableInformation, stackTraceAsArray);
+        }
+        EcsJsonSerializer.serializeObjectEnd(builder);
+        return builder.toString();
+    }
+
+    protected void setIncludeOrigin(final boolean includeOrigin) {
+        this.includeOrigin = includeOrigin;
+    }
+
+    protected void setServiceName(final String serviceName) {
+        this.serviceName = serviceName;
+    }
+
+    protected void setStackTraceAsArray(final boolean stackTraceAsArray) {
+        this.stackTraceAsArray = stackTraceAsArray;
+    }
+    
+    public void setEventDataset(String eventDataset) {
+        this.eventDataset = eventDataset;
+    }
+
+    private String getProperty(final String name, final String defaultValue) {
+        String value = LogManager.getLogManager().getProperty(name);
+        if (value == null) {
+            value = defaultValue;
+        } else {
+            value = value.trim();
+        }
+        return value;
+    }
+    
+    private String buildFileName(String className) {
+        String result = UNKNOWN_FILE;
+        if (className != null) {
+            int fileNameEnd = className.indexOf('$');
+            if (fileNameEnd < 0) {
+                fileNameEnd = className.length();
+            }
+            int classNameStart = className.lastIndexOf('.');
+            if (classNameStart < fileNameEnd) {
+                result = className.substring(classNameStart + 1, fileNameEnd) + ".java";
+            }
+        }
+        return result;
+    }
+
+}

jul-ecs-formatter/src/main/resources/META-INF/NOTICE

@@ -0,0 +1,2 @@
+ecs-logging-java
+Copyright 2019 - 2020 Elasticsearch B.V.