Skip to content

Commit 1eedce1

Browse files
lfranckelfrancke
committed
Backport HADOOP-18583 & fix OpenSSL native library
1 parent 4ceda06 commit 1eedce1

File tree

5 files changed

+243
-0
lines changed

5 files changed

+243
-0
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,7 @@ All notable changes to this project will be documented in this file.
5959
- zookeeper: bump jetty version for CVE-2024-13009 in 3.9.3 ([#1179])
6060
- zookeeper: bump netty version for CVE-2025-24970 in 3.9.3 ([#1180])
6161
- hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184])
62+
- hadoop: Backport HADOOP-18583 to make OpenSSL 3.x work with the native hadoop libraries ([#1209])
6263

6364
### Changed
6465

@@ -221,6 +222,7 @@ All notable changes to this project will be documented in this file.
221222
[#1188]: https://github.com/stackabletech/docker-images/pull/1188
222223
[#1189]: https://github.com/stackabletech/docker-images/pull/1189
223224
[#1197]: https://github.com/stackabletech/docker-images/pull/1197
225+
[#1209]: https://github.com/stackabletech/docker-images/pull/1209
224226

225227
## [25.3.0] - 2025-03-21
226228

‎hadoop/Dockerfile‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,15 @@ ln -s "/stackable/jmx/jmx_prometheus_javaagent-${JMX_EXPORTER}.jar" /stackable/j
136136
# Set correct permissions and ownerships
137137
chown --recursive ${STACKABLE_USER_UID}:0 /stackable/hadoop /stackable/jmx /stackable/async-profiler "/stackable/async-profiler-${ASYNC_PROFILER}-${TARGETOS}-${ARCH}"
138138
chmod --recursive g=u /stackable/jmx /stackable/async-profiler "/stackable/hadoop-${HADOOP_VERSION}-stackable${RELEASE}"
139+
140+
# Workaround for https://issues.apache.org/jira/browse/HADOOP-12845
141+
# The problem is that our stackable-devel image does contain the openssl-devel package
142+
# That package creates a symlink from /usr/lib/libcrypto.so to the real libcrypto
143+
# The non -devel package, which is used in this image, does NOT create this symlink.
144+
# That's why the Hadoop build works even with the 'require.openssl' flag but in the production
145+
# image the 'hadoop checknative' tool still fails because it can't find the 'libcrypto.so' symlink.
146+
# Therefore we create this symlink here.
147+
ln -s /usr/lib64/libcrypto.so.3 /usr/lib64/libcrypto.so
139148
EOF
140149

141150
RUN <<EOF

‎hadoop/hadoop/Dockerfile‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ sed -e '/<artifactId>hadoop-pipes<\/artifactId>/,/<\/dependency>/ { s/<version>.
6969
# Create snapshot of the source code including custom patches
7070
tar -czf /stackable/hadoop-${NEW_VERSION}-src.tar.gz .
7171

72+
# We do not pass require.snappy because that is only built in to the MapReduce client and we don't need that
7273
mvn \
7374
--batch-mode \
7475
--no-transfer-progress \
@@ -77,6 +78,7 @@ mvn \
7778
-pl '!hadoop-tools/hadoop-pipes' \
7879
-Dhadoop.version=${NEW_VERSION} \
7980
-Drequire.fuse=true \
81+
-Drequire.openssl=true \
8082
-DskipTests \
8183
-Dmaven.javadoc.skip=true
8284

‎hadoop/hadoop/stackable/patches/3.3.6/0012-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
From baa7ec826f3f6d044f5307efe4b5d3bdd111bf4e Mon Sep 17 00:00:00 2001
2+
From: Sebastian Klemke <[email protected]>
3+
Date: Thu, 7 Nov 2024 19:14:13 +0100
4+
Subject: HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256) (#7149)
5+
6+
Contributed by Sebastian Klemke
7+
---
8+
.../org/apache/hadoop/crypto/OpensslCipher.c | 68 +++++++++++++++++--
9+
1 file changed, 64 insertions(+), 4 deletions(-)
10+
11+
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
12+
index abff7ea5f1..f17169dec2 100644
13+
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
14+
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
15+
@@ -24,6 +24,57 @@
16+
17+
#include "org_apache_hadoop_crypto_OpensslCipher.h"
18+
19+
+/*
20+
+ # OpenSSL ABI Symbols
21+
+
22+
+ Available on all OpenSSL versions:
23+
+
24+
+ | Function | 1.0 | 1.1 | 3.0 |
25+
+ |--------------------------------|-----|-----|-----|
26+
+ | EVP_CIPHER_CTX_new | YES | YES | YES |
27+
+ | EVP_CIPHER_CTX_free | YES | YES | YES |
28+
+ | EVP_CIPHER_CTX_set_padding | YES | YES | YES |
29+
+ | EVP_CIPHER_CTX_test_flags | YES | YES | YES |
30+
+ | EVP_CipherInit_ex | YES | YES | YES |
31+
+ | EVP_CipherUpdate | YES | YES | YES |
32+
+ | EVP_CipherFinal_ex | YES | YES | YES |
33+
+ | ENGINE_by_id | YES | YES | YES |
34+
+ | ENGINE_free | YES | YES | YES |
35+
+ | EVP_aes_256_ctr | YES | YES | YES |
36+
+ | EVP_aes_128_ctr | YES | YES | YES |
37+
+
38+
+ Available on old versions:
39+
+
40+
+ | Function | 1.0 | 1.1 | 3.0 |
41+
+ |--------------------------------|-----|-----|-----|
42+
+ | EVP_CIPHER_CTX_cleanup | YES | --- | --- |
43+
+ | EVP_CIPHER_CTX_init | YES | --- | --- |
44+
+ | EVP_CIPHER_CTX_block_size | YES | YES | --- |
45+
+ | EVP_CIPHER_CTX_encrypting | --- | YES | --- |
46+
+
47+
+ Available on new versions:
48+
+
49+
+ | Function | 1.0 | 1.1 | 3.0 |
50+
+ |--------------------------------|-----|-----|-----|
51+
+ | OPENSSL_init_crypto | --- | YES | YES |
52+
+ | EVP_CIPHER_CTX_reset | --- | YES | YES |
53+
+ | EVP_CIPHER_CTX_get_block_size | --- | --- | YES |
54+
+ | EVP_CIPHER_CTX_is_encrypting | --- | --- | YES |
55+
+
56+
+ Optionally available on new versions:
57+
+
58+
+ | Function | 1.0 | 1.1 | 3.0 |
59+
+ |--------------------------------|-----|-----|-----|
60+
+ | EVP_sm4_ctr | --- | opt | opt |
61+
+
62+
+ Name changes:
63+
+
64+
+ | < 3.0 name | >= 3.0 name |
65+
+ |----------------------------|--------------------------------|
66+
+ | EVP_CIPHER_CTX_block_size | EVP_CIPHER_CTX_get_block_size |
67+
+ | EVP_CIPHER_CTX_encrypting | EVP_CIPHER_CTX_is_encrypting |
68+
+ */
69+
+
70+
#ifdef UNIX
71+
static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
72+
static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
73+
@@ -87,6 +138,15 @@ static __dlsym_EVP_aes_128_ctr dlsym_EVP_aes_128_ctr;
74+
static HMODULE openssl;
75+
#endif
76+
77+
+// names changed in OpenSSL 3 ABI - see History section in EVP_EncryptInit(3)
78+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
79+
+#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_get_block_size"
80+
+#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_is_encrypting"
81+
+#else
82+
+#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_block_size"
83+
+#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_encrypting"
84+
+#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
85+
+
86+
static void loadAesCtr(JNIEnv *env)
87+
{
88+
#ifdef UNIX
89+
@@ -142,10 +202,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
90+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \
91+
"EVP_CIPHER_CTX_test_flags");
92+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \
93+
- "EVP_CIPHER_CTX_block_size");
94+
+ CIPHER_CTX_BLOCK_SIZE);
95+
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
96+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \
97+
- "EVP_CIPHER_CTX_encrypting");
98+
+ CIPHER_CTX_ENCRYPTING);
99+
#endif
100+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \
101+
"EVP_CipherInit_ex");
102+
@@ -173,11 +233,11 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
103+
openssl, "EVP_CIPHER_CTX_test_flags");
104+
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \
105+
dlsym_EVP_CIPHER_CTX_block_size, env, \
106+
- openssl, "EVP_CIPHER_CTX_block_size");
107+
+ openssl, CIPHER_CTX_BLOCK_SIZE);
108+
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
109+
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \
110+
dlsym_EVP_CIPHER_CTX_encrypting, env, \
111+
- openssl, "EVP_CIPHER_CTX_encrypting");
112+
+ openssl, CIPHER_CTX_ENCRYPTING);
113+
#endif
114+
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \
115+
env, openssl, "EVP_CipherInit_ex");

‎hadoop/hadoop/stackable/patches/3.4.1/0011-HADOOP-18583.-Fix-loading-of-OpenSSL-3.x-symbols-525.patch‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
From cd1c23ea5bddd2796caf2590fef467e488c3bcbf Mon Sep 17 00:00:00 2001
2+
From: Sebastian Klemke <[email protected]>
3+
Date: Thu, 7 Nov 2024 19:14:13 +0100
4+
Subject: HADOOP-18583. Fix loading of OpenSSL 3.x symbols (#5256) (#7149)
5+
6+
Contributed by Sebastian Klemke
7+
---
8+
.../org/apache/hadoop/crypto/OpensslCipher.c | 68 +++++++++++++++++--
9+
1 file changed, 64 insertions(+), 4 deletions(-)
10+
11+
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
12+
index 976bf135ce..33be4a394f 100644
13+
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
14+
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/crypto/OpensslCipher.c
15+
@@ -24,6 +24,57 @@
16+
17+
#include "org_apache_hadoop_crypto_OpensslCipher.h"
18+
19+
+/*
20+
+ # OpenSSL ABI Symbols
21+
+
22+
+ Available on all OpenSSL versions:
23+
+
24+
+ | Function | 1.0 | 1.1 | 3.0 |
25+
+ |--------------------------------|-----|-----|-----|
26+
+ | EVP_CIPHER_CTX_new | YES | YES | YES |
27+
+ | EVP_CIPHER_CTX_free | YES | YES | YES |
28+
+ | EVP_CIPHER_CTX_set_padding | YES | YES | YES |
29+
+ | EVP_CIPHER_CTX_test_flags | YES | YES | YES |
30+
+ | EVP_CipherInit_ex | YES | YES | YES |
31+
+ | EVP_CipherUpdate | YES | YES | YES |
32+
+ | EVP_CipherFinal_ex | YES | YES | YES |
33+
+ | ENGINE_by_id | YES | YES | YES |
34+
+ | ENGINE_free | YES | YES | YES |
35+
+ | EVP_aes_256_ctr | YES | YES | YES |
36+
+ | EVP_aes_128_ctr | YES | YES | YES |
37+
+
38+
+ Available on old versions:
39+
+
40+
+ | Function | 1.0 | 1.1 | 3.0 |
41+
+ |--------------------------------|-----|-----|-----|
42+
+ | EVP_CIPHER_CTX_cleanup | YES | --- | --- |
43+
+ | EVP_CIPHER_CTX_init | YES | --- | --- |
44+
+ | EVP_CIPHER_CTX_block_size | YES | YES | --- |
45+
+ | EVP_CIPHER_CTX_encrypting | --- | YES | --- |
46+
+
47+
+ Available on new versions:
48+
+
49+
+ | Function | 1.0 | 1.1 | 3.0 |
50+
+ |--------------------------------|-----|-----|-----|
51+
+ | OPENSSL_init_crypto | --- | YES | YES |
52+
+ | EVP_CIPHER_CTX_reset | --- | YES | YES |
53+
+ | EVP_CIPHER_CTX_get_block_size | --- | --- | YES |
54+
+ | EVP_CIPHER_CTX_is_encrypting | --- | --- | YES |
55+
+
56+
+ Optionally available on new versions:
57+
+
58+
+ | Function | 1.0 | 1.1 | 3.0 |
59+
+ |--------------------------------|-----|-----|-----|
60+
+ | EVP_sm4_ctr | --- | opt | opt |
61+
+
62+
+ Name changes:
63+
+
64+
+ | < 3.0 name | >= 3.0 name |
65+
+ |----------------------------|--------------------------------|
66+
+ | EVP_CIPHER_CTX_block_size | EVP_CIPHER_CTX_get_block_size |
67+
+ | EVP_CIPHER_CTX_encrypting | EVP_CIPHER_CTX_is_encrypting |
68+
+ */
69+
+
70+
#ifdef UNIX
71+
static EVP_CIPHER_CTX * (*dlsym_EVP_CIPHER_CTX_new)(void);
72+
static void (*dlsym_EVP_CIPHER_CTX_free)(EVP_CIPHER_CTX *);
73+
@@ -106,6 +157,15 @@ static __dlsym_ENGINE_free dlsym_ENGINE_free;
74+
static HMODULE openssl;
75+
#endif
76+
77+
+// names changed in OpenSSL 3 ABI - see History section in EVP_EncryptInit(3)
78+
+#if OPENSSL_VERSION_NUMBER >= 0x30000000L
79+
+#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_get_block_size"
80+
+#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_is_encrypting"
81+
+#else
82+
+#define CIPHER_CTX_BLOCK_SIZE "EVP_CIPHER_CTX_block_size"
83+
+#define CIPHER_CTX_ENCRYPTING "EVP_CIPHER_CTX_encrypting"
84+
+#endif /* OPENSSL_VERSION_NUMBER >= 0x30000000L */
85+
+
86+
static void loadAesCtr(JNIEnv *env)
87+
{
88+
#ifdef UNIX
89+
@@ -170,10 +230,10 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
90+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_test_flags, env, openssl, \
91+
"EVP_CIPHER_CTX_test_flags");
92+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_block_size, env, openssl, \
93+
- "EVP_CIPHER_CTX_block_size");
94+
+ CIPHER_CTX_BLOCK_SIZE);
95+
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
96+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CIPHER_CTX_encrypting, env, openssl, \
97+
- "EVP_CIPHER_CTX_encrypting");
98+
+ CIPHER_CTX_ENCRYPTING);
99+
#endif
100+
LOAD_DYNAMIC_SYMBOL(dlsym_EVP_CipherInit_ex, env, openssl, \
101+
"EVP_CipherInit_ex");
102+
@@ -209,11 +269,11 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_crypto_OpensslCipher_initIDs
103+
openssl, "EVP_CIPHER_CTX_test_flags");
104+
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_block_size, \
105+
dlsym_EVP_CIPHER_CTX_block_size, env, \
106+
- openssl, "EVP_CIPHER_CTX_block_size");
107+
+ openssl, CIPHER_CTX_BLOCK_SIZE);
108+
#if OPENSSL_VERSION_NUMBER >= 0x10100000L
109+
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CIPHER_CTX_encrypting, \
110+
dlsym_EVP_CIPHER_CTX_encrypting, env, \
111+
- openssl, "EVP_CIPHER_CTX_encrypting");
112+
+ openssl, CIPHER_CTX_ENCRYPTING);
113+
#endif
114+
LOAD_DYNAMIC_SYMBOL(__dlsym_EVP_CipherInit_ex, dlsym_EVP_CipherInit_ex, \
115+
env, openssl, "EVP_CipherInit_ex");

0 commit comments

Comments
 (0)