From 203af5865dec6592de00cdf8f790f76cf8da4385 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 12 Feb 2025 14:13:23 +0100 Subject: [PATCH 01/12] Upgrade to Netty 4.1.118.Final --- build-tools-internal/version.properties | 2 +- gradle/verification-metadata.xml | 70 +++++++++++++++++++++++++ 2 files changed, 71 insertions(+), 1 deletion(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index a0c663b19a0c6..7a86706257690 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -14,7 +14,7 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 -netty = 4.1.115.Final +netty = 4.1.118.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 awsv1sdk = 1.12.270 diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 64efd8e439df1..060daea30c953 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1426,6 +1426,11 @@ + + + + + @@ -1436,6 +1441,11 @@ + + + + + @@ -1446,26 +1456,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + @@ -1476,6 +1511,11 @@ + + + + + @@ -1486,11 +1526,21 @@ + + + + + + + + + + @@ -1501,11 +1551,21 @@ + + + + + + + + + + @@ -1516,6 +1576,11 @@ + + + + + @@ -1526,6 +1591,11 @@ + + + + + From 2ead335b7a5c75133dd7a85694a4ccf95fb5a9d4 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 12 Feb 2025 16:03:27 +0100 Subject: [PATCH 02/12] Package tests check --- build-tools-internal/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index 7a86706257690..a0c663b19a0c6 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -14,7 +14,7 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 -netty = 4.1.118.Final +netty = 4.1.115.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 awsv1sdk = 1.12.270 From 9d6610ba7708d98d0bb7dd829bb518b4dc84fb25 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 12 Feb 2025 16:58:22 +0100 Subject: [PATCH 03/12] Failures unrelated --- build-tools-internal/version.properties | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-tools-internal/version.properties b/build-tools-internal/version.properties index a0c663b19a0c6..7a86706257690 100644 --- a/build-tools-internal/version.properties +++ b/build-tools-internal/version.properties @@ -14,7 +14,7 @@ log4j = 2.19.0 slf4j = 2.0.6 ecsLogging = 1.2.0 jna = 5.12.1 -netty = 4.1.115.Final +netty = 4.1.118.Final commons_lang3 = 3.9 google_oauth_client = 1.34.1 awsv1sdk = 1.12.270 From 028f1b1b8ca9073f7236e57806aed862b47519e6 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 14 Feb 2025 12:00:59 +0100 Subject: [PATCH 04/12] More logging in packaging tests --- .../java/org/elasticsearch/packaging/util/ServerUtils.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index ea71308b11940..2b1ee52aebacc 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -158,15 +158,18 @@ private static void waitForXpack(Installation installation) { int retries = 60; while (retries > 0) { retries -= 1; - try (Socket s = new Socket(InetAddress.getLoopbackAddress(), installation.port)) { + try (Socket ignored = new Socket(InetAddress.getLoopbackAddress(), installation.port)) { + logger.info("Connection established on retry {}", 60 - retries); return; } catch (IOException e) { // ignore, only want to establish a connection + logger.error("IOException while trying to connect to Elasticsearch", e); } try { Thread.sleep(2000); } catch (InterruptedException interrupted) { + logger.error("Interrupted!"); Thread.currentThread().interrupt(); return; } @@ -175,6 +178,8 @@ private static void waitForXpack(Installation installation) { FileUtils.logAllLogs(installation.logs, logger); } + logger.error("Elasticsearch (with x-pack) did not start"); + throw new RuntimeException("Elasticsearch (with x-pack) did not start"); } From f92fd6f7fe2d4898c684ec286f419a4bb8d0cdbf Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 14 Feb 2025 13:07:08 +0100 Subject: [PATCH 05/12] More --- .../java/org/elasticsearch/packaging/util/ServerUtils.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 2b1ee52aebacc..19b47495f301b 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -160,6 +160,9 @@ private static void waitForXpack(Installation installation) { retries -= 1; try (Socket ignored = new Socket(InetAddress.getLoopbackAddress(), installation.port)) { logger.info("Connection established on retry {}", 60 - retries); + if (installation != null) { + FileUtils.logAllLogs(installation.logs, logger); + } return; } catch (IOException e) { // ignore, only want to establish a connection From 9c3bc7628a88c9d2563147c2944f79177f1fcc9b Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 14 Feb 2025 13:20:35 +0100 Subject: [PATCH 06/12] Increase timeout --- .../src/test/java/org/elasticsearch/packaging/util/Shell.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java index 42348b1fe2644..15dbb36c66cdb 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java @@ -171,7 +171,7 @@ private Result runScriptIgnoreExitCode(String[] command) { try { Process process = builder.start(); - if (process.waitFor(10, TimeUnit.MINUTES) == false) { + if (process.waitFor(20, TimeUnit.MINUTES) == false) { if (process.isAlive()) { process.destroyForcibly(); } From 2f3bbaab5b8219cd2fefb29c4139076bcba4ed47 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Fri, 14 Feb 2025 16:04:20 +0100 Subject: [PATCH 07/12] Why security manager why --- .../org/elasticsearch/packaging/util/ServerUtils.java | 10 +--------- .../java/org/elasticsearch/packaging/util/Shell.java | 2 +- .../org/elasticsearch/bootstrap/security.policy | 2 ++ 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java index 19b47495f301b..ea71308b11940 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/ServerUtils.java @@ -158,21 +158,15 @@ private static void waitForXpack(Installation installation) { int retries = 60; while (retries > 0) { retries -= 1; - try (Socket ignored = new Socket(InetAddress.getLoopbackAddress(), installation.port)) { - logger.info("Connection established on retry {}", 60 - retries); - if (installation != null) { - FileUtils.logAllLogs(installation.logs, logger); - } + try (Socket s = new Socket(InetAddress.getLoopbackAddress(), installation.port)) { return; } catch (IOException e) { // ignore, only want to establish a connection - logger.error("IOException while trying to connect to Elasticsearch", e); } try { Thread.sleep(2000); } catch (InterruptedException interrupted) { - logger.error("Interrupted!"); Thread.currentThread().interrupt(); return; } @@ -181,8 +175,6 @@ private static void waitForXpack(Installation installation) { FileUtils.logAllLogs(installation.logs, logger); } - logger.error("Elasticsearch (with x-pack) did not start"); - throw new RuntimeException("Elasticsearch (with x-pack) did not start"); } diff --git a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java index 15dbb36c66cdb..42348b1fe2644 100644 --- a/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java +++ b/qa/packaging/src/test/java/org/elasticsearch/packaging/util/Shell.java @@ -171,7 +171,7 @@ private Result runScriptIgnoreExitCode(String[] command) { try { Process process = builder.start(); - if (process.waitFor(20, TimeUnit.MINUTES) == false) { + if (process.waitFor(10, TimeUnit.MINUTES) == false) { if (process.isAlive()) { process.destroyForcibly(); } diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 55abdc84fc8fb..664412cca5894 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -76,6 +76,8 @@ grant codeBase "${codebase.elasticsearch-simdvec}" { //// Everything else: grant { + permission java.lang.RuntimePermission "getClassLoader"; + // needed by vendored Guice permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.vm.annotation"; From bc28f1657c1789affe5ea977c16d1db124a47cc9 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 25 Feb 2025 16:33:29 +0100 Subject: [PATCH 08/12] Security policies --- .../src/main/plugin-metadata/plugin-security.policy | 1 + .../src/main/plugin-metadata/plugin-security.policy | 8 ++++++++ .../resources/org/elasticsearch/bootstrap/security.policy | 2 +- .../org/elasticsearch/bootstrap/test-framework.policy | 1 + .../src/main/plugin-metadata/plugin-security.policy | 1 + 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy index 8a7c623597376..5ba18621d58ee 100644 --- a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -12,6 +12,7 @@ grant { permission java.net.SocketPermission "*", "connect"; // io.netty.util.concurrent.GlobalEventExecutor.startThread permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; // Used by jackson bean deserialization permission java.lang.RuntimePermission "accessDeclaredMembers"; permission java.lang.reflect.ReflectPermission "suppressAccessChecks"; diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy index ed278af96d926..748645b39b78e 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -16,6 +16,9 @@ grant codeBase "${codebase.netty-common}" { // Netty sets custom classloader for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + + // Netty also gets the classloader for some of its internal threads + permission java.lang.RuntimePermission "getClassLoader"; }; grant codeBase "${codebase.netty-transport}" { @@ -23,3 +26,8 @@ grant codeBase "${codebase.netty-transport}" { // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; }; + +grant { + // Netty also gets the classloader for some of its internal threads + permission java.lang.RuntimePermission "getClassLoader"; +} diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 664412cca5894..4911708b27c4f 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -76,7 +76,7 @@ grant codeBase "${codebase.elasticsearch-simdvec}" { //// Everything else: grant { - permission java.lang.RuntimePermission "getClassLoader"; + // permission java.lang.RuntimePermission "getClassLoader"; // needed by vendored Guice permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.vm.annotation"; diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index ada61c118ec3c..d2bab71cdf06d 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -122,6 +122,7 @@ grant codeBase "${codebase.netty-common}" { permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; // Netty sets custom classloader for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; permission java.net.SocketPermission "*", "accept,connect"; }; diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index d814dfbb1c117..474b936f2a5c0 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -48,6 +48,7 @@ grant codeBase "${codebase.netty-common}" { permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; // Netty sets custom classloader for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; + permission java.lang.RuntimePermission "getClassLoader"; }; grant codeBase "${codebase.netty-transport}" { From 3d954a90d184b8a09e837ff1b546a1d1c93783fb Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 25 Feb 2025 16:37:16 +0100 Subject: [PATCH 09/12] Clean up --- .../src/main/plugin-metadata/plugin-security.policy | 1 + .../src/main/plugin-metadata/plugin-security.policy | 9 +-------- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy index 5ba18621d58ee..3aeeb6bde3914 100644 --- a/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy +++ b/modules/repository-azure/src/main/plugin-metadata/plugin-security.policy @@ -12,6 +12,7 @@ grant { permission java.net.SocketPermission "*", "connect"; // io.netty.util.concurrent.GlobalEventExecutor.startThread permission java.lang.RuntimePermission "setContextClassLoader"; + // io.netty.util.concurrent.GlobalEventExecutor.startThread permission java.lang.RuntimePermission "getClassLoader"; // Used by jackson bean deserialization permission java.lang.RuntimePermission "accessDeclaredMembers"; diff --git a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy index 748645b39b78e..dbf8e728c1606 100644 --- a/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy +++ b/modules/transport-netty4/src/main/plugin-metadata/plugin-security.policy @@ -14,10 +14,8 @@ grant codeBase "${codebase.netty-common}" { // netty makes and accepts socket connections permission java.net.SocketPermission "*", "accept,connect"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; - - // Netty also gets the classloader for some of its internal threads permission java.lang.RuntimePermission "getClassLoader"; }; @@ -26,8 +24,3 @@ grant codeBase "${codebase.netty-transport}" { // the bug says it only happened rarely, and that its fixed, but apparently it still happens rarely! permission java.util.PropertyPermission "sun.nio.ch.bugLevel", "write"; }; - -grant { - // Netty also gets the classloader for some of its internal threads - permission java.lang.RuntimePermission "getClassLoader"; -} From 4f15118606bbfcb6d4ac21f1d135cbeb0fd74f82 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Tue, 25 Feb 2025 16:42:13 +0100 Subject: [PATCH 10/12] Clean up verification metadata --- gradle/verification-metadata.xml | 110 ------------------ .../elasticsearch/bootstrap/security.policy | 2 - .../bootstrap/test-framework.policy | 2 +- .../plugin-metadata/plugin-security.policy | 2 +- 4 files changed, 2 insertions(+), 114 deletions(-) diff --git a/gradle/verification-metadata.xml b/gradle/verification-metadata.xml index 8de92877364d0..526891c7abb2e 100644 --- a/gradle/verification-metadata.xml +++ b/gradle/verification-metadata.xml @@ -1404,186 +1404,76 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy index 4911708b27c4f..55abdc84fc8fb 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/security.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/security.policy @@ -76,8 +76,6 @@ grant codeBase "${codebase.elasticsearch-simdvec}" { //// Everything else: grant { - // permission java.lang.RuntimePermission "getClassLoader"; - // needed by vendored Guice permission java.lang.RuntimePermission "accessClassInPackage.jdk.internal.vm.annotation"; diff --git a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy index d2bab71cdf06d..0e206a2005e74 100644 --- a/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy +++ b/server/src/main/resources/org/elasticsearch/bootstrap/test-framework.policy @@ -120,7 +120,7 @@ grant codeBase "${codebase.httpasyncclient}" { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; permission java.lang.RuntimePermission "getClassLoader"; permission java.net.SocketPermission "*", "accept,connect"; diff --git a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy index 474b936f2a5c0..b4791207a15bf 100644 --- a/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy +++ b/x-pack/plugin/security/src/main/plugin-metadata/plugin-security.policy @@ -46,7 +46,7 @@ grant { grant codeBase "${codebase.netty-common}" { // for reading the system-wide configuration for the backlog of established sockets permission java.io.FilePermission "/proc/sys/net/core/somaxconn", "read"; - // Netty sets custom classloader for some of its internal threads + // Netty gets and sets classloaders for some of its internal threads permission java.lang.RuntimePermission "setContextClassLoader"; permission java.lang.RuntimePermission "getClassLoader"; }; From 2d0a95889379f424e1d1ebcd05cafc97e11f2561 Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 26 Feb 2025 10:10:42 +0100 Subject: [PATCH 11/12] Update docs/changelog/122371.yaml --- docs/changelog/122371.yaml | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 docs/changelog/122371.yaml diff --git a/docs/changelog/122371.yaml b/docs/changelog/122371.yaml new file mode 100644 index 0000000000000..ba85e4d97c682 --- /dev/null +++ b/docs/changelog/122371.yaml @@ -0,0 +1,5 @@ +pr: 122371 +summary: Upgrade Netty to `4.1.118.Final` +area: Network +type: upgrade +issues: [] From 8a6cd3eeedf8667144c6e0420b47a981b4e2ea8c Mon Sep 17 00:00:00 2001 From: Nikolaj Volgushev Date: Wed, 26 Feb 2025 10:27:27 +0100 Subject: [PATCH 12/12] Delete docs/changelog/122371.yaml --- docs/changelog/122371.yaml | 5 ----- 1 file changed, 5 deletions(-) delete mode 100644 docs/changelog/122371.yaml diff --git a/docs/changelog/122371.yaml b/docs/changelog/122371.yaml deleted file mode 100644 index ba85e4d97c682..0000000000000 --- a/docs/changelog/122371.yaml +++ /dev/null @@ -1,5 +0,0 @@ -pr: 122371 -summary: Upgrade Netty to `4.1.118.Final` -area: Network -type: upgrade -issues: []