From 39e94fab4765ab420591776d405a2531c427d4d9 Mon Sep 17 00:00:00 2001 From: Ramanathan Date: Wed, 27 Aug 2025 14:22:17 +0530 Subject: [PATCH 1/7] Bump elasticsearch-java to version 9.x #3663 --- log4j-layout-template-json-test/pom.xml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index 6f77cb724ab..702dba4c6a4 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -48,7 +48,7 @@ 2. The Docker image version of the ELK-stack As of 2024-09-16, these all (Maven artifacts and Elastic products) get released with the same version. --> - 8.17.3 + 9.1.3 @@ -132,6 +132,13 @@ test + + org.elasticsearch.client + elasticsearch-rest-client + 8.13.4 + test + + From a1a9a45196974aaa6ffaab0d94d096fde30a7e2d Mon Sep 17 00:00:00 2001 From: Ramanathan Date: Fri, 5 Sep 2025 23:33:55 +0530 Subject: [PATCH 2/7] reverted back to elastic.version to 8.17.3 and add maven-toolchains-plugin configuration --- log4j-layout-template-json-test/pom.xml | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index 702dba4c6a4..ad8be7ab1dc 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -48,7 +48,7 @@ 2. The Docker image version of the ELK-stack As of 2024-09-16, these all (Maven artifacts and Elastic products) get released with the same version. --> - 9.1.3 + 8.17.3 @@ -196,6 +196,26 @@ + + org.apache.maven.plugins + maven-toolchains-plugin + 3.1.0 + + + + 17 + + + + + + + toolchain + + + + + io.fabric8 docker-maven-plugin From b01d12d8007e4e33806c665a58e8eddb0d795594 Mon Sep 17 00:00:00 2001 From: Ramanathan Date: Sat, 6 Sep 2025 14:52:19 +0530 Subject: [PATCH 3/7] #3663 Update elastic.version to 9.1.3 and add maven-toolchains-plugin configuration --- log4j-layout-template-json-test/pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index ad8be7ab1dc..02237ca2d42 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -48,7 +48,7 @@ 2. The Docker image version of the ELK-stack As of 2024-09-16, these all (Maven artifacts and Elastic products) get released with the same version. --> - 8.17.3 + 9.1.3 From 49d7d52f6a19c74ecc7f03abd1add9820d55aace Mon Sep 17 00:00:00 2001 From: Ramanathan Date: Sat, 6 Sep 2025 23:24:56 +0530 Subject: [PATCH 4/7] #3663 Refactor LogstashIT to use Elasticsearch REST client and remove deprecated dependencies --- log4j-layout-template-json-test/pom.xml | 41 ++------- .../layout/template/json/LogstashIT.java | 86 ++++++------------- 2 files changed, 36 insertions(+), 91 deletions(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index 02237ca2d42..669f654dfd3 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -84,15 +84,15 @@ - org.awaitility - awaitility + org.elasticsearch.client + elasticsearch-rest-client + ${elastic.version} test - co.elastic.clients - elasticsearch-java - ${elastic.version} + org.awaitility + awaitility test @@ -132,13 +132,6 @@ test - - org.elasticsearch.client - elasticsearch-rest-client - 8.13.4 - test - - @@ -196,26 +189,6 @@ - - org.apache.maven.plugins - maven-toolchains-plugin - 3.1.0 - - - - 17 - - - - - - - toolchain - - - - - io.fabric8 docker-maven-plugin @@ -387,6 +360,10 @@ ${logstash.gelf.port} ${logstash.tcp.port} + + 17 + + diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java index f0c553b1611..a788a0f3f50 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java @@ -19,19 +19,8 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; -import co.elastic.clients.elasticsearch.ElasticsearchClient; -import co.elastic.clients.elasticsearch._types.ElasticsearchException; -import co.elastic.clients.elasticsearch._types.HealthStatus; -import co.elastic.clients.elasticsearch.cluster.HealthResponse; -import co.elastic.clients.elasticsearch.core.CountResponse; -import co.elastic.clients.elasticsearch.core.SearchResponse; -import co.elastic.clients.elasticsearch.core.search.SourceConfig; -import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; -import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse; -import co.elastic.clients.json.jackson.JacksonJsonpMapper; -import co.elastic.clients.transport.ElasticsearchTransport; -import co.elastic.clients.transport.rest_client.RestClientTransport; import co.elastic.logging.log4j2.EcsLayout; +import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -63,6 +52,9 @@ import org.apache.logging.log4j.message.SimpleMessage; import org.apache.logging.log4j.status.StatusLogger; import org.apache.logging.log4j.util.Strings; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -136,10 +128,7 @@ class LogstashIT { private static final String ES_INDEX_MESSAGE_FIELD_NAME = "message"; private static RestClient REST_CLIENT; - - private static ElasticsearchTransport ES_TRANSPORT; - - private static ElasticsearchClient ES_CLIENT; + private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); /** * Constants hardcoded in `docker-maven-plugin` configuration, do not change! @@ -182,16 +171,16 @@ static void initEsClient() { final String hostUri = String.format("http://%s:%d", MavenHardcodedConstants.HOST_NAME, MavenHardcodedConstants.ES_PORT); REST_CLIENT = RestClient.builder(HttpHost.create(hostUri)).build(); - ES_TRANSPORT = new RestClientTransport(REST_CLIENT, new JacksonJsonpMapper()); - ES_CLIENT = new ElasticsearchClient(ES_TRANSPORT); - LOGGER.info(LOG_PREFIX + "verifying the ES connection to `{}`", hostUri); await("ES cluster health") .pollDelay(100, TimeUnit.MILLISECONDS) .atMost(1, TimeUnit.MINUTES) .untilAsserted(() -> { - final HealthResponse healthResponse = ES_CLIENT.cluster().health(); - assertThat(healthResponse.status()).isNotEqualTo(HealthStatus.Red); + Request request = new Request("GET", "/_cluster/health"); + Response response = REST_CLIENT.performRequest(request); + Map health = + OBJECT_MAPPER.readValue(response.getEntity().getContent(), Map.class); + assertThat(health.get("status")).isNotEqualTo("red"); }); } @@ -217,11 +206,9 @@ private static void waitForSocketBinding(final int port, final String name) { void deleteIndex() throws IOException { LOGGER.info(LOG_PREFIX + "deleting the ES index"); try { - DeleteIndexResponse deleteIndexResponse = ES_CLIENT - .indices() - .delete(DeleteIndexRequest.of(builder -> builder.index(MavenHardcodedConstants.ES_INDEX_NAME))); - assertThat(deleteIndexResponse.acknowledged()).isTrue(); - } catch (ElasticsearchException error) { + Request request = new Request("DELETE", "/" + MavenHardcodedConstants.ES_INDEX_NAME); + REST_CLIENT.performRequest(request); + } catch (ResponseException error) { if (!error.getMessage().contains("index_not_found_exception")) { throw new RuntimeException(error); } @@ -230,7 +217,6 @@ void deleteIndex() throws IOException { @AfterAll static void stopClient() throws Exception { - ES_TRANSPORT.close(); REST_CLIENT.close(); } @@ -466,42 +452,24 @@ private static SocketAppender createStartedAppender(final Layout layout, fina } private static void assertDocumentCount(final int expectedCount) throws IOException { - final CountResponse countResponse; - try { - countResponse = ES_CLIENT.count(builder -> builder.index(MavenHardcodedConstants.ES_INDEX_NAME)); - } - // Try to enrich the failure with the available list of indices - catch (final ElasticsearchException error) { - try { - if (error.getMessage().contains("index_not_found_exception")) { - final Set indexNames = - ES_CLIENT.cluster().health().indices().keySet(); - final String message = String.format("Could not find index! Available index names: %s", indexNames); - throw new AssertionError(message, error); - } - } catch (final Exception suppressed) { - error.addSuppressed(suppressed); - } - throw error; - } - final long actualCount = countResponse.count(); + Request request = new Request("GET", "/" + MavenHardcodedConstants.ES_INDEX_NAME + "/_count"); + Response response = REST_CLIENT.performRequest(request); + Map countResponse = + OBJECT_MAPPER.readValue(response.getEntity().getContent(), Map.class); + long actualCount = ((Number) countResponse.get("count")).longValue(); assertThat(actualCount).isEqualTo(expectedCount); } private static List> queryDocuments() throws IOException { - @SuppressWarnings("rawtypes") - SearchResponse searchResponse = ES_CLIENT.search( - searchBuilder -> searchBuilder - .index(MavenHardcodedConstants.ES_INDEX_NAME) - .size(LOG_EVENT_COUNT) - .source(SourceConfig.of(sourceConfigBuilder -> sourceConfigBuilder.fetch(true))), - Map.class); - return searchResponse.hits().hits().stream() - .map(hit -> { - @SuppressWarnings("unchecked") - final Map source = hit.source(); - return source; - }) + Request request = + new Request("GET", "/" + MavenHardcodedConstants.ES_INDEX_NAME + "/_search?size=" + LOG_EVENT_COUNT); + Response response = REST_CLIENT.performRequest(request); + Map searchResponse = + OBJECT_MAPPER.readValue(response.getEntity().getContent(), Map.class); + List> hits = + (List>) ((Map) searchResponse.get("hits")).get("hits"); + return hits.stream() + .map(hit -> (Map) hit.get("_source")) .collect(Collectors.toList()); } } From 36ad178dc550c263bd4bb7e9d2680da5ae6de213 Mon Sep 17 00:00:00 2001 From: Ramanathan Date: Fri, 12 Sep 2025 06:43:18 +0530 Subject: [PATCH 5/7] issue #3663 Reorder dependencies in pom.xml to replace awaitility Signed-off-by: Ramanathan --- log4j-layout-template-json-test/pom.xml | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index 669f654dfd3..4f03c74c059 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -84,15 +84,15 @@ - org.elasticsearch.client - elasticsearch-rest-client - ${elastic.version} + org.awaitility + awaitility test - org.awaitility - awaitility + org.elasticsearch.client + elasticsearch-rest-client + ${elastic.version} test @@ -360,9 +360,6 @@ ${logstash.gelf.port} ${logstash.tcp.port} - - 17 - From 386ebd132a7233a09d9dd5a774098be432317f89 Mon Sep 17 00:00:00 2001 From: Ramanathan Date: Wed, 1 Oct 2025 00:31:59 +0530 Subject: [PATCH 6/7] issue-3663 Refactor LogstashIT to use Elasticsearch Java client and update pom.xml dependencies Signed-off-by: Ramanathan --- log4j-layout-template-json-test/pom.xml | 40 ++++++--- .../layout/template/json/LogstashIT.java | 86 +++++++++++++------ 2 files changed, 88 insertions(+), 38 deletions(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index 4f03c74c059..f9a667a5526 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -37,17 +37,17 @@ true + ~ OSGi and JPMS options + --> org.apache.logging.log4j.layout.template.json.test org.apache.logging.log4j.core + As of 2024-09-16, these all (Maven artifacts and Elastic products) get released with the same version. --> 9.1.3 @@ -89,6 +89,13 @@ test + + co.elastic.clients + elasticsearch-java + ${elastic.version} + test + + org.elasticsearch.client elasticsearch-rest-client @@ -96,6 +103,18 @@ test + + org.apache.httpcomponents + httpcore + test + + + + org.apache.httpcomponents + httpclient + test + + com.fasterxml.jackson.core jackson-databind @@ -142,8 +161,8 @@ maven-surefire-plugin + indeed handles everything in UTF-8 without implicitly + relying on the system defaults. --> -Dfile.encoding=US-ASCII @@ -162,8 +181,8 @@ docker + ~ Only the `ubuntu` CI runners have access to Docker + --> linux @@ -181,7 +200,7 @@ false + We limit the used JVM memory to avoid this OOM failure. --> -Xms750m -Xmx750m @@ -360,7 +379,6 @@ ${logstash.gelf.port} ${logstash.tcp.port} - diff --git a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java index a788a0f3f50..f0c553b1611 100644 --- a/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java +++ b/log4j-layout-template-json-test/src/test/java/org/apache/logging/log4j/layout/template/json/LogstashIT.java @@ -19,8 +19,19 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.awaitility.Awaitility.await; +import co.elastic.clients.elasticsearch.ElasticsearchClient; +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import co.elastic.clients.elasticsearch._types.HealthStatus; +import co.elastic.clients.elasticsearch.cluster.HealthResponse; +import co.elastic.clients.elasticsearch.core.CountResponse; +import co.elastic.clients.elasticsearch.core.SearchResponse; +import co.elastic.clients.elasticsearch.core.search.SourceConfig; +import co.elastic.clients.elasticsearch.indices.DeleteIndexRequest; +import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse; +import co.elastic.clients.json.jackson.JacksonJsonpMapper; +import co.elastic.clients.transport.ElasticsearchTransport; +import co.elastic.clients.transport.rest_client.RestClientTransport; import co.elastic.logging.log4j2.EcsLayout; -import com.fasterxml.jackson.databind.ObjectMapper; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.PrintStream; @@ -52,9 +63,6 @@ import org.apache.logging.log4j.message.SimpleMessage; import org.apache.logging.log4j.status.StatusLogger; import org.apache.logging.log4j.util.Strings; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.client.ResponseException; import org.elasticsearch.client.RestClient; import org.junit.jupiter.api.AfterAll; import org.junit.jupiter.api.BeforeAll; @@ -128,7 +136,10 @@ class LogstashIT { private static final String ES_INDEX_MESSAGE_FIELD_NAME = "message"; private static RestClient REST_CLIENT; - private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper(); + + private static ElasticsearchTransport ES_TRANSPORT; + + private static ElasticsearchClient ES_CLIENT; /** * Constants hardcoded in `docker-maven-plugin` configuration, do not change! @@ -171,16 +182,16 @@ static void initEsClient() { final String hostUri = String.format("http://%s:%d", MavenHardcodedConstants.HOST_NAME, MavenHardcodedConstants.ES_PORT); REST_CLIENT = RestClient.builder(HttpHost.create(hostUri)).build(); + ES_TRANSPORT = new RestClientTransport(REST_CLIENT, new JacksonJsonpMapper()); + ES_CLIENT = new ElasticsearchClient(ES_TRANSPORT); + LOGGER.info(LOG_PREFIX + "verifying the ES connection to `{}`", hostUri); await("ES cluster health") .pollDelay(100, TimeUnit.MILLISECONDS) .atMost(1, TimeUnit.MINUTES) .untilAsserted(() -> { - Request request = new Request("GET", "/_cluster/health"); - Response response = REST_CLIENT.performRequest(request); - Map health = - OBJECT_MAPPER.readValue(response.getEntity().getContent(), Map.class); - assertThat(health.get("status")).isNotEqualTo("red"); + final HealthResponse healthResponse = ES_CLIENT.cluster().health(); + assertThat(healthResponse.status()).isNotEqualTo(HealthStatus.Red); }); } @@ -206,9 +217,11 @@ private static void waitForSocketBinding(final int port, final String name) { void deleteIndex() throws IOException { LOGGER.info(LOG_PREFIX + "deleting the ES index"); try { - Request request = new Request("DELETE", "/" + MavenHardcodedConstants.ES_INDEX_NAME); - REST_CLIENT.performRequest(request); - } catch (ResponseException error) { + DeleteIndexResponse deleteIndexResponse = ES_CLIENT + .indices() + .delete(DeleteIndexRequest.of(builder -> builder.index(MavenHardcodedConstants.ES_INDEX_NAME))); + assertThat(deleteIndexResponse.acknowledged()).isTrue(); + } catch (ElasticsearchException error) { if (!error.getMessage().contains("index_not_found_exception")) { throw new RuntimeException(error); } @@ -217,6 +230,7 @@ void deleteIndex() throws IOException { @AfterAll static void stopClient() throws Exception { + ES_TRANSPORT.close(); REST_CLIENT.close(); } @@ -452,24 +466,42 @@ private static SocketAppender createStartedAppender(final Layout layout, fina } private static void assertDocumentCount(final int expectedCount) throws IOException { - Request request = new Request("GET", "/" + MavenHardcodedConstants.ES_INDEX_NAME + "/_count"); - Response response = REST_CLIENT.performRequest(request); - Map countResponse = - OBJECT_MAPPER.readValue(response.getEntity().getContent(), Map.class); - long actualCount = ((Number) countResponse.get("count")).longValue(); + final CountResponse countResponse; + try { + countResponse = ES_CLIENT.count(builder -> builder.index(MavenHardcodedConstants.ES_INDEX_NAME)); + } + // Try to enrich the failure with the available list of indices + catch (final ElasticsearchException error) { + try { + if (error.getMessage().contains("index_not_found_exception")) { + final Set indexNames = + ES_CLIENT.cluster().health().indices().keySet(); + final String message = String.format("Could not find index! Available index names: %s", indexNames); + throw new AssertionError(message, error); + } + } catch (final Exception suppressed) { + error.addSuppressed(suppressed); + } + throw error; + } + final long actualCount = countResponse.count(); assertThat(actualCount).isEqualTo(expectedCount); } private static List> queryDocuments() throws IOException { - Request request = - new Request("GET", "/" + MavenHardcodedConstants.ES_INDEX_NAME + "/_search?size=" + LOG_EVENT_COUNT); - Response response = REST_CLIENT.performRequest(request); - Map searchResponse = - OBJECT_MAPPER.readValue(response.getEntity().getContent(), Map.class); - List> hits = - (List>) ((Map) searchResponse.get("hits")).get("hits"); - return hits.stream() - .map(hit -> (Map) hit.get("_source")) + @SuppressWarnings("rawtypes") + SearchResponse searchResponse = ES_CLIENT.search( + searchBuilder -> searchBuilder + .index(MavenHardcodedConstants.ES_INDEX_NAME) + .size(LOG_EVENT_COUNT) + .source(SourceConfig.of(sourceConfigBuilder -> sourceConfigBuilder.fetch(true))), + Map.class); + return searchResponse.hits().hits().stream() + .map(hit -> { + @SuppressWarnings("unchecked") + final Map source = hit.source(); + return source; + }) .collect(Collectors.toList()); } } From 561107e6eeba7e404fd473fab2e5a951d5e72368 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Volkan=20Yaz=C4=B1c=C4=B1?= Date: Fri, 3 Oct 2025 15:02:46 +0200 Subject: [PATCH 7/7] Remove redundant dependencies and changes --- log4j-layout-template-json-test/pom.xml | 32 ++++++++----------------- 1 file changed, 10 insertions(+), 22 deletions(-) diff --git a/log4j-layout-template-json-test/pom.xml b/log4j-layout-template-json-test/pom.xml index f9a667a5526..00da24c198e 100644 --- a/log4j-layout-template-json-test/pom.xml +++ b/log4j-layout-template-json-test/pom.xml @@ -37,17 +37,17 @@ true + ~ OSGi and JPMS options + --> org.apache.logging.log4j.layout.template.json.test org.apache.logging.log4j.core + As of 2024-09-16, these all (Maven artifacts and Elastic products) get released with the same version. --> 9.1.3 @@ -103,18 +103,6 @@ test - - org.apache.httpcomponents - httpcore - test - - - - org.apache.httpcomponents - httpclient - test - - com.fasterxml.jackson.core jackson-databind @@ -161,8 +149,8 @@ maven-surefire-plugin + indeed handles everything in UTF-8 without implicitly + relying on the system defaults. --> -Dfile.encoding=US-ASCII @@ -181,8 +169,8 @@ docker + ~ Only the `ubuntu` CI runners have access to Docker + --> linux @@ -200,7 +188,7 @@ false + We limit the used JVM memory to avoid this OOM failure. --> -Xms750m -Xmx750m