Initial logging module prototype

Author: ChrisHegarty

benchmarks/src/main/java/org/elasticsearch/benchmark/fs/AvailableIndexFoldersBenchmark.java

@@ -7,10 +7,10 @@
  */
 package org.elasticsearch.benchmark.fs;
 
-import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.env.Environment;
 import org.elasticsearch.env.NodeEnvironment;
+import org.elasticsearch.logging.internal.LogConfigurator;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
 import org.openjdk.jmh.annotations.Fork;

libs/build.gradle

@@ -24,6 +24,7 @@ configure(subprojects - project('elasticsearch-log4j')) {
         if (depProject != null
           && false == depProject.path.equals(':libs:elasticsearch-x-content')
           && false == depProject.path.equals(':libs:elasticsearch-core')
+          && false == depProject.path.equals(':libs:elasticsearch-cli')
           && depProject.path.startsWith(':libs')
           && depProject.name.startsWith('elasticsearch-')) {
           throw new InvalidUserDataException("projects in :libs "

libs/logging/build.gradle

@@ -0,0 +1,40 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+apply plugin: 'elasticsearch.publish'
+
+dependencies {
+  moduleCompileOnly  project(':libs:elasticsearch-x-content')  // TODO: for JsonStringEncoder
+  moduleCompileOnly  project(':libs:elasticsearch-cli')  // TODO: prob remove this, if just for exception types
+  moduleCompileOnly "org.apache.logging.log4j:log4j-api:${versions.log4j}"
+  moduleCompileOnly "org.apache.logging.log4j:log4j-core:${versions.log4j}"
+
+  moduleCompileOnly "co.elastic.logging:log4j2-ecs-layout:${versions.ecsLogging}"
+  moduleCompileOnly "co.elastic.logging:ecs-logging-core:${versions.ecsLogging}"
+
+  testImplementation "com.carrotsearch.randomizedtesting:randomizedtesting-runner:${versions.randomizedrunner}"
+  testImplementation "junit:junit:${versions.junit}"
+  testImplementation "org.hamcrest:hamcrest:${versions.hamcrest}"
+
+  testImplementation(project(":test:framework")) {
+    exclude group: 'org.elasticsearch', module: 'elasticsearch-logging'
+  }
+}
+
+//tasks.named('forbiddenApisMain').configure {
+//  // :libs:elasticsearch-core does not depend on server
+//  // TODO: Need to decide how we want to handle for forbidden signatures with the changes to server
+//  replaceSignatureFiles 'jdk-signatures'
+//}
+//
+tasks.named("compileJava").configure {
+  options.compilerArgs.add("-Xlint:-requires-automatic")
+  options.compilerArgs.add("-Xlint:-module") // qualified exports
+  options.compilerArgs.add("-Xlint:-exports") // implements Message!!
+}
+

libs/logging/src/main/java/module-info.java

@@ -0,0 +1,21 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+module org.elasticsearch.logging {
+    requires org.elasticsearch.cli;
+    requires org.elasticsearch.xcontent;
+    requires log4j2.ecs.layout;
+    requires ecs.logging.core;
+    requires org.apache.logging.log4j;
+    requires org.apache.logging.log4j.core;
+
+    exports org.elasticsearch.logging;
+    exports org.elasticsearch.logging.internal to org.elasticsearch.server;
+    exports org.elasticsearch.logging.internal.spi to org.elasticsearch.server;
+
+}

server/src/main/java/org/elasticsearch/common/logging/DeprecatedMessage.java renamed to libs/logging/src/main/java/org/elasticsearch/logging/DeprecatedMessage.java

@@ -6,25 +6,28 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.common.logging;
+package org.elasticsearch.logging;
 
-import org.elasticsearch.common.Strings;
-import org.elasticsearch.common.SuppressLoggerChecks;
+//import org.elasticsearch.common.SuppressLoggerChecks;
 
 import java.util.Locale;
 
+import static org.elasticsearch.logging.internal.Util.isNullOrEmpty;
+
 /**
  * A logger message used by {@link DeprecationLogger}, enriched with fields
  * named following ECS conventions. Carries x-opaque-id field if provided in the headers.
  * Will populate the x-opaque-id field in JSON logs.
  */
-public class DeprecatedMessage {
+public final class DeprecatedMessage {
     public static final String ELASTIC_ORIGIN_FIELD_NAME = "elasticsearch.elastic_product_origin";
     public static final String KEY_FIELD_NAME = "event.code";
     public static final String X_OPAQUE_ID_FIELD_NAME = "elasticsearch.http.request.x_opaque_id";
     public static final String ECS_VERSION = "1.2.0";
 
-    @SuppressLoggerChecks(reason = "safely delegates to logger")
+    private DeprecatedMessage() {}
+
+    // @SuppressLoggerChecks(reason = "safely delegates to logger")
     public static ESLogMessage of(
         DeprecationCategory category,
         String key,
@@ -36,7 +39,7 @@ public static ESLogMessage of(
         return getEsLogMessage(category, key, xOpaqueId, productOrigin, messagePattern, args);
     }
 
-    @SuppressLoggerChecks(reason = "safely delegates to logger")
+    // @SuppressLoggerChecks(reason = "safely delegates to logger")
     public static ESLogMessage compatibleDeprecationMessage(
         String key,
         String xOpaqueId,
@@ -47,7 +50,7 @@ public static ESLogMessage compatibleDeprecationMessage(
         return getEsLogMessage(DeprecationCategory.COMPATIBLE_API, key, xOpaqueId, productOrigin, messagePattern, args);
     }
 
-    @SuppressLoggerChecks(reason = "safely delegates to logger")
+    // @SuppressLoggerChecks(reason = "safely delegates to logger")
     private static ESLogMessage getEsLogMessage(
         DeprecationCategory category,
         String key,
@@ -62,7 +65,7 @@ private static ESLogMessage getEsLogMessage(
             .field(KEY_FIELD_NAME, key)
             .field("elasticsearch.event.category", category.name().toLowerCase(Locale.ROOT));
 
-        if (Strings.isNullOrEmpty(xOpaqueId)) {
+        if (isNullOrEmpty(xOpaqueId)) {
             return esLogMessage;
         }
 

server/src/main/java/org/elasticsearch/common/logging/DeprecationCategory.java renamed to libs/logging/src/main/java/org/elasticsearch/logging/DeprecationCategory.java

@@ -6,7 +6,7 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.common.logging;
+package org.elasticsearch.logging;
 
 /**
  * Deprecation log messages are categorised so that consumers of the logs can easily aggregate them.

server/src/main/java/org/elasticsearch/common/logging/DeprecationLogger.java renamed to libs/logging/src/main/java/org/elasticsearch/logging/DeprecationLogger.java

@@ -6,11 +6,11 @@
  * Side Public License, v 1.
  */
 
-package org.elasticsearch.common.logging;
+package org.elasticsearch.logging;
 
-import org.apache.logging.log4j.Level;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
+import org.elasticsearch.logging.internal.HeaderWarningAppender;
+import org.elasticsearch.logging.internal.RateLimitingFilter;
+import org.elasticsearch.logging.internal.ServerSupportImpl;
 
 import java.security.AccessController;
 import java.security.PrivilegedAction;
@@ -29,12 +29,12 @@
  * key is combined with the <code>X-Opaque-Id</code> request header value, if supplied, which allows for per-client
  * message limiting.
  */
-public class DeprecationLogger {
+public final class DeprecationLogger {
     /**
      * Deprecation messages are logged at this level.
      * More serious that WARN by 1, but less serious than ERROR
      */
-    public static Level CRITICAL = Level.forName("CRITICAL", Level.WARN.intLevel() - 1);
+    public static Level CRITICAL = Level.of("CRITICAL", Level.WARN.getSeverity() - 1);
 
     private final Logger logger;
 
@@ -97,8 +97,8 @@ public DeprecationLogger warn(final DeprecationCategory category, final String k
     private DeprecationLogger logDeprecation(Level level, DeprecationCategory category, String key, String msg, Object[] params) {
         assert category != DeprecationCategory.COMPATIBLE_API
             : "DeprecationCategory.COMPATIBLE_API should be logged with compatibleApiWarning method";
-        String opaqueId = HeaderWarning.getXOpaqueId();
-        String productOrigin = HeaderWarning.getProductOrigin();
+        String opaqueId = ServerSupportImpl.INSTANCE.getXOpaqueIdHeader();
+        String productOrigin = ServerSupportImpl.INSTANCE.getProductOriginHeader();
         ESLogMessage deprecationMessage = DeprecatedMessage.of(category, key, opaqueId, productOrigin, msg, params);
         doPrivilegedLog(level, deprecationMessage);
         return this;
@@ -130,8 +130,8 @@ public DeprecationLogger compatibleCritical(final String key, final String msg,
      * so that it can be returned to the client.
      */
     public DeprecationLogger compatible(final Level level, final String key, final String msg, final Object... params) {
-        String opaqueId = HeaderWarning.getXOpaqueId();
-        String productOrigin = HeaderWarning.getProductOrigin();
+        String opaqueId = ServerSupportImpl.INSTANCE.getXOpaqueIdHeader();
+        String productOrigin = ServerSupportImpl.INSTANCE.getProductOriginHeader();
         ESLogMessage deprecationMessage = DeprecatedMessage.compatibleDeprecationMessage(key, opaqueId, productOrigin, msg, params);
         logger.log(level, deprecationMessage);
         return this;

libs/logging/src/main/java/org/elasticsearch/logging/ESLogMessage.java

@@ -0,0 +1,80 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logging;
+
+import org.elasticsearch.logging.internal.ESLogMessageImpl;
+
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+/**
+ * Custom logger messages. Carries additional fields which will populate JSON fields in logs.
+ */
+public final class ESLogMessage implements Message {
+
+    private final ESLogMessageImpl impl;
+
+    public ESLogMessage() {
+        impl = new ESLogMessageImpl();
+    }
+
+    public ESLogMessage(String messagePattern, Object... args) {
+        impl = new ESLogMessageImpl(messagePattern, args);
+
+    }
+
+    public ESLogMessage argAndField(String key, Object value) {
+        return this.argAndField(key, value);
+    }
+
+    public ESLogMessage withFields(Map<String, Object> jsonFields) {
+        return this.withFields(jsonFields);
+    }
+
+    public ESLogMessage field(String key, Object value) {
+        return this.field(key, value);
+    }
+
+    public String get(String key) {
+        return impl.get(key);
+    }
+
+    @Override
+    public String getFormattedMessage() {
+        return impl.getFormattedMessage();
+    }
+
+    @Override
+    public String getFormat() {
+        return impl.getFormat();
+    }
+
+    @Override
+    public Object[] getParameters() {
+        return impl.getParameters();
+    }
+
+    @Override
+    public Throwable getThrowable() {
+        return impl.getThrowable();
+    }
+
+    public static String asJsonArray(Stream<String> stream) {
+        return "[" + stream.map(ESLogMessageImpl::inQuotes).collect(Collectors.joining(", ")) + "]";
+    }
+
+    // static ESLogMessage of() {
+    // return new ESLogMessageImpl();
+    // }
+    //
+    // static ESLogMessage of(String messagePattern, Object... args) {
+    // return new ESLogMessageImpl(messagePattern, args);
+    // }
+}

libs/logging/src/main/java/org/elasticsearch/logging/Level.java

@@ -0,0 +1,81 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logging;
+
+import org.elasticsearch.logging.internal.StandardLevels;
+
+import java.util.Locale;
+import java.util.Objects;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+public final class Level {
+
+    public static final Level OFF = new Level("OFF", StandardLevels.OFF);
+
+    public static final Level FATAL = new Level("FATAL", StandardLevels.FATAL);
+
+    public static final Level ERROR = new Level("ERROR", StandardLevels.ERROR);
+
+    public static final Level WARN = new Level("WARN", StandardLevels.WARN);
+
+    public static final Level INFO = new Level("INFO", StandardLevels.INFO);
+
+    public static final Level DEBUG = new Level("DEBUG", StandardLevels.DEBUG);
+
+    public static final Level TRACE = new Level("TRACE", StandardLevels.TRACE);
+
+    public static final Level ALL = new Level("ALL", StandardLevels.ALL);
+
+    private static final ConcurrentMap<String, Level> LEVELS = new ConcurrentHashMap<>();
+
+    private final String name;
+
+    private final int severity;
+
+    /*package*/ static Level of(String name, int severity) {
+        var level = new Level(name, severity);
+        if (LEVELS.putIfAbsent(name, level) != null) {
+            throw new IllegalStateException("Level " + name + " is already been defined.");
+        }
+        return level;
+    }
+
+    private Level(String name, int severity) {
+        this.name = name;
+        this.severity = severity;
+    }
+
+    /**
+     * Returns the name of this level.
+     */
+    public String name() {
+        return name;
+    }
+
+    public int getSeverity() {
+        return severity;
+    }
+
+    /** Return the Level associated with the name. */
+    public static Level valueOf(final String name) {
+        Objects.requireNonNull(name);
+        final String levelName = name.trim().toUpperCase(Locale.ROOT);
+        final Level level = LEVELS.get(levelName);
+        if (level != null) {
+            return level;
+        }
+        throw new IllegalArgumentException("Unknown level constant [" + levelName + "].");
+    }
+
+    public boolean isMoreSpecificThan(Level level) {
+        return this.severity <= level.severity;
+    }
+
+}

libs/logging/src/main/java/org/elasticsearch/logging/LogManager.java

@@ -0,0 +1,23 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+package org.elasticsearch.logging;
+
+public class LogManager {
+
+    public static Logger getLogger(final String name) {
+        return null;
+    }
+
+    public static Logger getLogger(final Class<?> clazz) {
+        return null;
+    }
+
+    private LogManager() {}
+
+}