diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 82d5645af232e..f03f13cb91085 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -10,7 +10,7 @@ jackson = 2.15.0 snakeyaml = 2.0 icu4j = 77.1 supercsv = 2.4.0 -log4j = 2.19.0 +log4j = 2.25.1 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 diff --git a/docs/changelog/132166.yaml b/docs/changelog/132166.yaml new file mode 100644 index 0000000000000..ad3348fe926f3 --- /dev/null +++ b/docs/changelog/132166.yaml @@ -0,0 +1,6 @@ +pr: 132166 +summary: Upgrade to log4j 2.25.1 +area: Infra/Logging +type: upgrade +issues: + - 132035 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index b4b17c450200f..748598b22cee8 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -502,6 +502,11 @@ + + + + + @@ -687,6 +692,11 @@ + + + + + @@ -3110,6 +3120,11 @@ + + + + + @@ -3175,11 +3190,21 @@ + + + + + + + + + + diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties index 255c53a8d7dd8..a8e7ce57d6329 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/config/log4j2.properties @@ -30,7 +30,7 @@ appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning logger.deprecation.name = deprecation -logger.deprecation.level = deprecation +logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file logger.deprecation.appenderRef.header_warning.ref = header_warning logger.deprecation.additivity = false diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties index e488da2f95f58..5f7d4dbe5347c 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/deprecation/log4j2.properties @@ -24,7 +24,7 @@ appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning logger.deprecation.name = deprecation -logger.deprecation.level = deprecation +logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file logger.deprecation.appenderRef.header_warning.ref = header_warning logger.deprecation.additivity = false diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties index 1a1f70c7a2657..ce749ecbdfd05 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/no_node_name/log4j2.properties @@ -24,7 +24,7 @@ appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning logger.deprecation.name = deprecation -logger.deprecation.level = deprecation +logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file logger.deprecation.appenderRef.header_warning.ref = header_warning logger.deprecation.additivity = false diff --git a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties index 284b7bda1f7b0..7bf19d2b41875 100644 --- a/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties +++ b/qa/evil-tests/src/test/resources/org/elasticsearch/common/logging/settings/log4j2.properties @@ -24,7 +24,7 @@ appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning logger.deprecation.name = org.elasticsearch.deprecation.common.settings -logger.deprecation.level = deprecation +logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_console.ref = console logger.deprecation.appenderRef.deprecation_file.ref = deprecation_file logger.deprecation.appenderRef.header_warning.ref = header_warning diff --git a/qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties b/qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties index d45bfc6826d6a..2793177a3ea75 100644 --- a/qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties +++ b/qa/logging-config/src/javaRestTest/resources/es-v7-log4j2.properties @@ -102,7 +102,7 @@ appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning ################################################# logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = deprecation +logger.deprecation.level = warn logger.deprecation.appenderRef.deprecation_rolling.ref = deprecation_rolling logger.deprecation.appenderRef.deprecation_rolling_old.ref = deprecation_rolling_old logger.deprecation.appenderRef.header_warning.ref = header_warning diff --git a/server/build.gradle b/server/build.gradle index 20557aefba731..2bcd6e5322d21 100644 --- a/server/build.gradle +++ b/server/build.gradle @@ -202,12 +202,12 @@ tasks.named("thirdPartyAudit").configure { 'com.fasterxml.jackson.databind.ser.impl.SimpleFilterProvider', 'com.fasterxml.jackson.databind.ser.std.StdScalarSerializer', 'com.fasterxml.jackson.databind.ser.std.StdSerializer', + 'com.fasterxml.jackson.databind.util.ClassUtil', 'com.fasterxml.jackson.dataformat.xml.JacksonXmlModule', 'com.fasterxml.jackson.dataformat.xml.XmlMapper', 'com.fasterxml.jackson.dataformat.xml.util.DefaultXmlPrettyPrinter', - 'org.fusesource.jansi.Ansi', - 'org.fusesource.jansi.AnsiRenderer$Code', 'com.lmax.disruptor.EventFactory', + 'com.lmax.disruptor.EventHandler', 'com.lmax.disruptor.EventTranslator', 'com.lmax.disruptor.EventTranslatorTwoArg', 'com.lmax.disruptor.EventTranslatorVararg', @@ -246,6 +246,7 @@ tasks.named("thirdPartyAudit").configure { 'org.apache.commons.csv.QuoteMode', 'org.apache.kafka.clients.producer.Producer', 'org.apache.kafka.clients.producer.RecordMetadata', + 'org.apache.kafka.common.serialization.ByteArraySerializer', 'org.codehaus.stax2.XMLStreamWriter2', 'org.jctools.queues.MpscArrayQueue', 'org.osgi.framework.Bundle', @@ -257,13 +258,24 @@ tasks.named("thirdPartyAudit").configure { 'org.osgi.framework.ServiceReference', 'org.osgi.framework.ServiceRegistration', 'org.osgi.framework.SynchronousBundleListener', + 'org.osgi.framework.wiring.BundleRevision', 'org.osgi.framework.wiring.BundleWire', 'org.osgi.framework.wiring.BundleWiring', + 'org.zeromq.SocketType', + 'org.zeromq.ZContext', + 'org.zeromq.ZMonitor', + 'org.zeromq.ZMonitor$Event', + 'org.zeromq.ZMonitor$ZEvent', 'org.zeromq.ZMQ$Context', 'org.zeromq.ZMQ$Socket', 'org.zeromq.ZMQ', ) ignoreMissingClasses 'javax.xml.bind.DatatypeConverter' + + ignoreViolations( + 'org.apache.logging.log4j.core.util.internal.UnsafeUtil', + 'org.apache.logging.log4j.core.util.internal.UnsafeUtil$1', + ) } tasks.named("dependencyLicenses").configure { diff --git a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java index 4117d3c33efa4..3d2f8bfe53247 100644 --- a/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java +++ b/server/src/main/java/org/elasticsearch/action/search/SearchPhaseExecutionException.java @@ -26,21 +26,26 @@ public class SearchPhaseExecutionException extends ElasticsearchException { private final String phaseName; private final ShardSearchFailure[] shardFailures; + private final ElasticsearchException guessedCause; // log4j requires a stable cause! public SearchPhaseExecutionException(String phaseName, String msg, ShardSearchFailure[] shardFailures) { this(phaseName, msg, null, shardFailures); } + @SuppressWarnings("this-escape") public SearchPhaseExecutionException(String phaseName, String msg, Throwable cause, ShardSearchFailure[] shardFailures) { super(msg, deduplicateCause(cause, shardFailures)); this.phaseName = phaseName; this.shardFailures = shardFailures; + this.guessedCause = cause == null || super.getCause() == null ? guessFirstRootCause(shardFailures) : null; } + @SuppressWarnings("this-escape") public SearchPhaseExecutionException(StreamInput in) throws IOException { super(in); phaseName = in.readOptionalString(); shardFailures = in.readArray(ShardSearchFailure::readShardSearchFailure, ShardSearchFailure[]::new); + guessedCause = super.getCause() == null ? guessFirstRootCause(shardFailures) : null; } @Override @@ -96,14 +101,9 @@ public ShardSearchFailure[] shardFailures() { @Override public Throwable getCause() { + // note: log4j requires this to return a stable, consistent cause when called multiple times Throwable cause = super.getCause(); - if (cause == null) { - // fall back to guessed root cause - for (ElasticsearchException rootCause : guessRootCauses()) { - return rootCause; - } - } - return cause; + return cause != null ? cause : guessedCause; } @Override @@ -144,6 +144,16 @@ public ElasticsearchException[] guessRootCauses() { return rootCauses.toArray(new ElasticsearchException[0]); } + private static ElasticsearchException guessFirstRootCause(ShardSearchFailure[] shardFailures) { + for (ShardOperationFailedException failure : shardFailures) { + ElasticsearchException[] rootCauses = ElasticsearchException.guessRootCauses(failure.getCause()); + if (rootCauses.length > 0) { + return rootCauses[0]; + } + } + return null; + } + @Override public String toString() { return "Failed to execute phase [" diff --git a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java index 8150456194b76..1919b615d9035 100644 --- a/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java +++ b/test/framework/src/main/java/org/elasticsearch/test/ESTestCase.java @@ -152,9 +152,11 @@ import org.elasticsearch.xcontent.XContentParser.Token; import org.elasticsearch.xcontent.XContentParserConfiguration; import org.elasticsearch.xcontent.XContentType; +import org.hamcrest.Description; import org.hamcrest.Matcher; import org.hamcrest.MatcherAssert; import org.hamcrest.Matchers; +import org.hamcrest.TypeSafeMatcher; import org.junit.After; import org.junit.AfterClass; import org.junit.Before; @@ -223,10 +225,9 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.common.util.CollectionUtils.arrayAsArrayList; import static org.hamcrest.Matchers.anyOf; -import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.empty; -import static org.hamcrest.Matchers.emptyCollectionOf; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.everyItem; import static org.hamcrest.Matchers.hasItem; import static org.hamcrest.Matchers.startsWith; @@ -823,11 +824,20 @@ public void log(StatusData data) { // Tolerate the absence or otherwise denial of these specific lookup classes. // At some future time, we should require the JDNI warning. - private static final List LOG_4J_MSG_PREFIXES = List.of( - "JNDI lookup class is not available because this JRE does not support JNDI. " - + "JNDI string lookups will not be available, continuing configuration.", - "JMX runtime input lookup class is not available because this JRE does not support JMX. " - + "JMX lookups will not be available, continuing configuration. " + private static final Matcher LOG_4J_MSG_PREFIXES = anyOf( + startsWith( + "JNDI lookup class is not available because this JRE does not support JNDI. " + + "JNDI string lookups will not be available, continuing configuration." + ), + startsWith( + "JMX runtime input lookup class is not available because this JRE does not support JMX. " + + "JMX lookups will not be available, continuing configuration. " + ), + // TODO migrate to annotation processor https://github.com/elastic/elasticsearch/issues/135022 + startsWith("The use of package scanning to locate Log4j plugins is deprecated."), + startsWith("Some custom `Core` Log4j plugins are not properly registered"), + startsWith("Some custom `Converter` Log4j plugins are not properly registered"), + startsWith("No Root logger was configured, creating default ERROR-level Root logger with Console appender") ); // separate method so that this can be checked again after suite scoped cluster is shut down @@ -836,16 +846,25 @@ protected static void checkStaticState() throws Exception { assertThat(StatusLogger.getLogger().getLevel(), equalTo(Level.WARN)); synchronized (statusData) { try { - // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; we map the - // StatusData instances to Strings as otherwise their toString output is useless - assertThat( - statusData.stream().map(status -> status.getMessage().getFormattedMessage()).collect(Collectors.toList()), - anyOf( - emptyCollectionOf(String.class), - contains(startsWith(LOG_4J_MSG_PREFIXES.get(0)), startsWith(LOG_4J_MSG_PREFIXES.get(1))), - contains(startsWith(LOG_4J_MSG_PREFIXES.get(1))) - ) - ); + // ensure that there are no status logger messages which would indicate a problem with our Log4j usage; + assertThat(statusData, everyItem(new TypeSafeMatcher() { + @Override + protected boolean matchesSafely(StatusData item) { + return LOG_4J_MSG_PREFIXES.matches(item.getFormattedStatus()); + } + + @Override + public void describeTo(Description description) { + LOG_4J_MSG_PREFIXES.describeTo(description); + } + + @Override + protected void describeMismatchSafely(StatusData item, Description mismatchDescription) { + // make sure we see log4j exceptions in case of issues + mismatchDescription.appendText("was ").appendValue(item.getFormattedStatus()); + } + })); + } finally { // we clear the list so that status data from other tests do not interfere with tests within the same JVM statusData.clear(); diff --git a/test/framework/src/main/resources/log4j2-test.properties b/test/framework/src/main/resources/log4j2-test.properties index a78bb3ab2dfad..e8076cab58b5f 100644 --- a/test/framework/src/main/resources/log4j2-test.properties +++ b/test/framework/src/main/resources/log4j2-test.properties @@ -10,5 +10,5 @@ appender.header_warning.type = HeaderWarningAppender appender.header_warning.name = header_warning logger.deprecation.name = org.elasticsearch.deprecation -logger.deprecation.level = deprecation +logger.deprecation.level = warn logger.deprecation.appenderRef.header_warning.ref = header_warning diff --git a/x-pack/plugin/core/build.gradle b/x-pack/plugin/core/build.gradle index 1c5a081948bad..c80ae9a6b91a9 100644 --- a/x-pack/plugin/core/build.gradle +++ b/x-pack/plugin/core/build.gradle @@ -32,7 +32,7 @@ esplugin { tasks.named("dependencyLicenses").configure { mapping from: /http.*/, to: 'httpclient' // pulled in by rest client mapping from: /commons-.*/, to: 'commons' // pulled in by rest client - mapping from: /slf4j-.*/, to: 'slf4j' + mapping from: /slf4j-.*/, to: 'slf4j' } configurations { @@ -157,8 +157,6 @@ tasks.named("thirdPartyAudit").configure { 'javax.jms.Message', // HttpClient5 can use Conscrypt (TLS using BoringSSL), but we don't want that 'org.conscrypt.Conscrypt', - // SLF4j via HttpClient5 - 'org.slf4j.ext.EventData' ) } diff --git a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java index 8844fec09cf53..bd71882bb10a4 100644 --- a/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java +++ b/x-pack/plugin/core/src/main/java/org/elasticsearch/xpack/core/security/support/NoOpLogger.java @@ -10,6 +10,7 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.message.EntryMessage; +import org.apache.logging.log4j.message.FlowMessageFactory; import org.apache.logging.log4j.message.Message; import org.apache.logging.log4j.message.MessageFactory; import org.apache.logging.log4j.util.MessageSupplier; @@ -930,6 +931,11 @@ public MF getMessageFactory() { return null; } + @Override + public FlowMessageFactory getFlowMessageFactory() { + return null; + } + @Override public String getName() { return null; diff --git a/x-pack/plugin/ent-search/build.gradle b/x-pack/plugin/ent-search/build.gradle index b9802566b6c2f..c9f39759cb2fc 100644 --- a/x-pack/plugin/ent-search/build.gradle +++ b/x-pack/plugin/ent-search/build.gradle @@ -63,13 +63,6 @@ tasks.named("dependencyLicenses") { mapping from: /jackson.*/, to: 'jackson' } -tasks.named("thirdPartyAudit") { - ignoreMissingClasses( - // [missing classes] SLF4j includes an optional class that depends on an extension class (!) - 'org.slf4j.ext.EventData' - ) -} - tasks.named("yamlRestTest") { usesDefaultDistribution("uses the xpack/usage api") } diff --git a/x-pack/plugin/identity-provider/build.gradle b/x-pack/plugin/identity-provider/build.gradle index 6478694110e00..0e318666521fb 100644 --- a/x-pack/plugin/identity-provider/build.gradle +++ b/x-pack/plugin/identity-provider/build.gradle @@ -249,8 +249,6 @@ tasks.named("thirdPartyAudit").configure { 'org.bouncycastle.operator.jcajce.JcaContentSignerBuilder', 'org.bouncycastle.util.Arrays', 'org.bouncycastle.util.io.Streams', - // SLF4j - 'org.slf4j.ext.EventData' ) ignoreViolations( diff --git a/x-pack/plugin/security/build.gradle b/x-pack/plugin/security/build.gradle index e42ec82c5f8b4..6309d9c739cc4 100644 --- a/x-pack/plugin/security/build.gradle +++ b/x-pack/plugin/security/build.gradle @@ -426,8 +426,6 @@ tasks.named("thirdPartyAudit").configure { 'javax.activation.DataSource', 'javax.activation.FileDataSource', 'javax.activation.FileTypeMap', - // SLF4j - 'org.slf4j.ext.EventData' ) }