diff --git a/CHANGELOG.md b/CHANGELOG.md index bb98d4b37..8bf0d8695 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -59,6 +59,7 @@ All notable changes to this project will be documented in this file. - zookeeper: bump jetty version for CVE-2024-13009 in 3.9.3 ([#1179]) - zookeeper: bump netty version for CVE-2025-24970 in 3.9.3 ([#1180]) - hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184]) +- hadoop: Backport HADOOP-18583 to make OpenSSL 3.x work with the native hadoop libraries ([#1209]). ### Changed @@ -221,6 +222,7 @@ All notable changes to this project will be documented in this file. [#1188]: https://github.com/stackabletech/docker-images/pull/1188 [#1189]: https://github.com/stackabletech/docker-images/pull/1189 [#1197]: https://github.com/stackabletech/docker-images/pull/1197 +[#1209]: https://github.com/stackabletech/docker-images/pull/1209 ## [25.3.0] - 2025-03-21 diff --git a/hadoop/Dockerfile b/hadoop/Dockerfile index 2df24243d..5647d6c57 100644 --- a/hadoop/Dockerfile +++ b/hadoop/Dockerfile @@ -136,6 +136,15 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j # Set correct permissions and ownerships chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}" chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}" + +# Workaround for https://issues.apache.org/jira/browse/HADOOP-12845 +# The problem is that our stackable-devel image does contain the openssl-devel package +# That package creates a symlink from /usr/lib/libcrypto.so to the real libcrypto +# The non -devel package, which is used in this image, does NOT create this symlink. +# That's why the Hadoop build works even with the 'require.openssl' flag but in the production +# image the 'hadoop checknative' tool still fails because it can't find the 'libcrypto.so' symlink. +# Therefore we create this symlink here. +ln -s /usr/lib64/libcrypto.so.3 /usr/lib64/libcrypto.so EOF RUN <hadoop-pipes<\/artifactId>/,/<\/dependency>/ { s/. # Create snapshot of the source code including custom patches tar -czf /stackable/hadoop-${NEW_VERSION}-src.tar.gz . +# We do not pass require.snappy because that is only built in to the MapReduce client and we don't need that +# +# Passing require.openssl SHOULD make the build fail if OpenSSL is not present. +# This does not work properly however because this builder image contains the openssl-devel package which creates a symlink from /usr/lib64/libcrypto.so to the real version. +# Therefore, this build does work but the final image does NOT contain the openssl-devel package which is why it fails there which is why we have to create the symlink over there manually. +# We still leave this flag in to automatically fail should anything with the packages or symlinks ever fail. mvn \ --batch-mode \ --no-transfer-progress \ @@ -77,6 +83,7 @@ mvn \ -pl '!hadoop-tools/hadoop-pipes' \ -Dhadoop.version=${NEW_VERSION} \ -Drequire.fuse=true \ + -Drequire.openssl=true \ -DskipTests \ -Dmaven.javadoc.skip=true diff --git a/hadoop/hadoop/stackable/patches/3.3.6/0012-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch b/hadoop/hadoop/stackable/patches/3.3.6/0012-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch new file mode 100644 index 000000000..97c1eac8d --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.3.6/0012-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch @@ -0,0 +1,115 @@ +From baa7ec826f3f6d044f5307efe4b5d3bdd111bf4e Mon Sep 17 00:00:00 2001 +From: Sebastian Klemke <3669903+packet23@users.noreply.github.com> +Date: Thu, 7 Nov 2024 19:14:13 +0100 +Subject: HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256) (#7149) + +Contributed by Sebastian Klemke +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 68 +++++++++++++++++-- + 1 file changed, 64 insertions(+), 4 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index abff7ea5f1..f17169dec2 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -24,6 +24,57 @@ + + #include "org_apache_hadoop_crypto_OpensslCipher.h" + ++/* ++ # OpenSSL ABI Symbols ++ ++ Available on all OpenSSL versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | EVP_CIPHER_CTX_new | YES | YES | YES | ++ | EVP_CIPHER_CTX_free | YES | YES | YES | ++ | EVP_CIPHER_CTX_set_padding | YES | YES | YES | ++ | EVP_CIPHER_CTX_test_flags | YES | YES | YES | ++ | EVP_CipherInit_ex | YES | YES | YES | ++ | EVP_CipherUpdate | YES | YES | YES | ++ | EVP_CipherFinal_ex | YES | YES | YES | ++ | ENGINE_by_id | YES | YES | YES | ++ | ENGINE_free | YES | YES | YES | ++ | EVP_aes_256_ctr | YES | YES | YES | ++ | EVP_aes_128_ctr | YES | YES | YES | ++ ++ Available on old versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | EVP_CIPHER_CTX_cleanup | YES | --- | --- | ++ | EVP_CIPHER_CTX_init | YES | --- | --- | ++ | EVP_CIPHER_CTX_block_size | YES | YES | --- | ++ | EVP_CIPHER_CTX_encrypting | --- | YES | --- | ++ ++ Available on new versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | OPENSSL_init_crypto | --- | YES | YES | ++ | EVP_CIPHER_CTX_reset | --- | YES | YES | ++ | EVP_CIPHER_CTX_get_block_size | --- | --- | YES | ++ | EVP_CIPHER_CTX_is_encrypting | --- | --- | YES | ++ ++ Optionally available on new versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | EVP_sm4_ctr | --- | opt | opt | ++ ++ Name changes: ++ ++ | < 3.0 name | >= 3.0 name | ++ |----------------------------|--------------------------------| ++ | EVP_CIPHER_CTX_block_size | EVP_CIPHER_CTX_get_block_size | ++ | EVP_CIPHER_CTX_encrypting | EVP_CIPHER_CTX_is_encrypting | ++ */ ++ + #ifdef UNIX + static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void); + static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); +@@ -87,6 +138,15 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr; + static HMODULE openssl; + #endif + ++// names changed in OpenSSL 3 ABI - see History section in EVP_EncryptInit(3) ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_get_block_size" ++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_is_encrypting" ++#else ++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_block_size" ++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_encrypting" ++#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */ ++ + static void loadAesCtr(JNIEnv *env) + { + #ifdef UNIX +@@ -142,10 +202,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \ + "EVP_CIPHER_CTX_test_flags"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \ +- "EVP_CIPHER_CTX_block_size"); ++ CIPHER_CTX_BLOCK_SIZE); + #if OPENSSL_VERSION_NUMBER >= 0x10100000L + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \ +- "EVP_CIPHER_CTX_encrypting"); ++ CIPHER_CTX_ENCRYPTING); + #endif + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \ + "EVP_CipherInit_ex"); +@@ -173,11 +233,11 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + openssl, "EVP_CIPHER_CTX_test_flags"); + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \ + dlsym_EVP_CIPHER_CTX_block_size, env, \ +- openssl, "EVP_CIPHER_CTX_block_size"); ++ openssl, CIPHER_CTX_BLOCK_SIZE); + #if OPENSSL_VERSION_NUMBER >= 0x10100000L + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \ + dlsym_EVP_CIPHER_CTX_encrypting, env, \ +- openssl, "EVP_CIPHER_CTX_encrypting"); ++ openssl, CIPHER_CTX_ENCRYPTING); + #endif + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \ + env, openssl, "EVP_CipherInit_ex"); diff --git a/hadoop/hadoop/stackable/patches/3.4.1/0011-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch b/hadoop/hadoop/stackable/patches/3.4.1/0011-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch new file mode 100644 index 000000000..bc331db1f --- /dev/null +++ b/hadoop/hadoop/stackable/patches/3.4.1/0011-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch @@ -0,0 +1,115 @@ +From cd1c23ea5bddd2796caf2590fef467e488c3bcbf Mon Sep 17 00:00:00 2001 +From: Sebastian Klemke <3669903+packet23@users.noreply.github.com> +Date: Thu, 7 Nov 2024 19:14:13 +0100 +Subject: HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256) (#7149) + +Contributed by Sebastian Klemke +--- + .../org/apache/hadoop/crypto/OpensslCipher.c | 68 +++++++++++++++++-- + 1 file changed, 64 insertions(+), 4 deletions(-) + +diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +index 976bf135ce..33be4a394f 100644 +--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c ++++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c +@@ -24,6 +24,57 @@ + + #include "org_apache_hadoop_crypto_OpensslCipher.h" + ++/* ++ # OpenSSL ABI Symbols ++ ++ Available on all OpenSSL versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | EVP_CIPHER_CTX_new | YES | YES | YES | ++ | EVP_CIPHER_CTX_free | YES | YES | YES | ++ | EVP_CIPHER_CTX_set_padding | YES | YES | YES | ++ | EVP_CIPHER_CTX_test_flags | YES | YES | YES | ++ | EVP_CipherInit_ex | YES | YES | YES | ++ | EVP_CipherUpdate | YES | YES | YES | ++ | EVP_CipherFinal_ex | YES | YES | YES | ++ | ENGINE_by_id | YES | YES | YES | ++ | ENGINE_free | YES | YES | YES | ++ | EVP_aes_256_ctr | YES | YES | YES | ++ | EVP_aes_128_ctr | YES | YES | YES | ++ ++ Available on old versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | EVP_CIPHER_CTX_cleanup | YES | --- | --- | ++ | EVP_CIPHER_CTX_init | YES | --- | --- | ++ | EVP_CIPHER_CTX_block_size | YES | YES | --- | ++ | EVP_CIPHER_CTX_encrypting | --- | YES | --- | ++ ++ Available on new versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | OPENSSL_init_crypto | --- | YES | YES | ++ | EVP_CIPHER_CTX_reset | --- | YES | YES | ++ | EVP_CIPHER_CTX_get_block_size | --- | --- | YES | ++ | EVP_CIPHER_CTX_is_encrypting | --- | --- | YES | ++ ++ Optionally available on new versions: ++ ++ | Function | 1.0 | 1.1 | 3.0 | ++ |--------------------------------|-----|-----|-----| ++ | EVP_sm4_ctr | --- | opt | opt | ++ ++ Name changes: ++ ++ | < 3.0 name | >= 3.0 name | ++ |----------------------------|--------------------------------| ++ | EVP_CIPHER_CTX_block_size | EVP_CIPHER_CTX_get_block_size | ++ | EVP_CIPHER_CTX_encrypting | EVP_CIPHER_CTX_is_encrypting | ++ */ ++ + #ifdef UNIX + static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void); + static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *); +@@ -106,6 +157,15 @@ static __dlsym_ENGINE_free dlsym_ENGINE_free; + static HMODULE openssl; + #endif + ++// names changed in OpenSSL 3 ABI - see History section in EVP_EncryptInit(3) ++#if OPENSSL_VERSION_NUMBER >= 0x30000000L ++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_get_block_size" ++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_is_encrypting" ++#else ++#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_block_size" ++#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_encrypting" ++#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */ ++ + static void loadAesCtr(JNIEnv *env) + { + #ifdef UNIX +@@ -170,10 +230,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \ + "EVP_CIPHER_CTX_test_flags"); + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \ +- "EVP_CIPHER_CTX_block_size"); ++ CIPHER_CTX_BLOCK_SIZE); + #if OPENSSL_VERSION_NUMBER >= 0x10100000L + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \ +- "EVP_CIPHER_CTX_encrypting"); ++ CIPHER_CTX_ENCRYPTING); + #endif + LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \ + "EVP_CipherInit_ex"); +@@ -209,11 +269,11 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs + openssl, "EVP_CIPHER_CTX_test_flags"); + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \ + dlsym_EVP_CIPHER_CTX_block_size, env, \ +- openssl, "EVP_CIPHER_CTX_block_size"); ++ openssl, CIPHER_CTX_BLOCK_SIZE); + #if OPENSSL_VERSION_NUMBER >= 0x10100000L + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \ + dlsym_EVP_CIPHER_CTX_encrypting, env, \ +- openssl, "EVP_CIPHER_CTX_encrypting"); ++ openssl, CIPHER_CTX_ENCRYPTING); + #endif + LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \ + env, openssl, "EVP_CipherInit_ex");