Skip to content

Commit aa50852

Browse files
dervoetidervoeti
committed
Merge branch 'main' of https://github.com/stackabletech/docker-images into feat/separate-hadoop-dockerfile
2 parents d2c7cd7 + 0bb7f02 commit aa50852

12 files changed

+629
-23
lines changed

‎CHANGELOG.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,7 @@ All notable changes to this project will be documented in this file.
5656
- git-sync: Bump version to 4.4.1 ([#1151]).
5757
- zookeeper: bump jetty version for CVE-2024-13009 in 3.9.3 ([#1179])
5858
- zookeeper: bump netty version for CVE-2025-24970 in 3.9.3 ([#1180])
59+
- hadoop: backport HADOOP-19352, HADOOP-19335, HADOOP-19465, HADOOP-19456 and HADOOP-19225 to fix vulnerabilities in Hadoop `3.4.1` ([#1184])
5960

6061
### Changed
6162

@@ -88,6 +89,7 @@ All notable changes to this project will be documented in this file.
8889
- zookeeper: Enable custom versions ([#1169]).
8990
- opa: Enable custom versions ([#1170]).
9091
- use custom product versions for Hadoop, HBase, Phoenix, hbase-operator-tools, Druid, Hive and Spark ([#1173]).
92+
- hbase: Bump dependencies to the latest patch level for HBase `2.6.1` and `2.6.2` ([#1185]).
9193
- hadoop: Separate Dockerfiles for Hadoop build and HDFS image ([#1186]).
9294

9395
### Fixed
@@ -203,6 +205,8 @@ All notable changes to this project will be documented in this file.
203205
[#1174]: https://github.com/stackabletech/docker-images/pull/1174
204206
[#1179]: https://github.com/stackabletech/docker-images/pull/1179
205207
[#1180]: https://github.com/stackabletech/docker-images/pull/1180
208+
[#1184]: https://github.com/stackabletech/docker-images/pull/1184
209+
[#1185]: https://github.com/stackabletech/docker-images/pull/1185
206210
[#1186]: https://github.com/stackabletech/docker-images/pull/1186
207211

208212
## [25.3.0] - 2025-03-21

‎hadoop/hadoop/stackable/patches/3.4.1/0006-HADOOP-19352.-Hadoop-OSS-Connector-adds-support-for-.patch‎

Lines changed: 247 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,247 @@
1+
From 699f329ba20d938004fc2b983b7b225de36ecf88 Mon Sep 17 00:00:00 2001
2+
From: =?UTF-8?q?=E5=91=A8=E7=BF=B1?= <[email protected]>
3+
Date: Tue, 18 Feb 2025 14:16:37 +0800
4+
Subject: HADOOP-19352. Hadoop OSS Connector adds support for V4 signatures.
5+
(#7205)
6+
7+
* Aliyun oss connector support v4 signature
8+
---
9+
hadoop-project/pom.xml | 2 +-
10+
hadoop-tools/hadoop-aliyun/pom.xml | 26 +++++
11+
.../aliyun/oss/AliyunOSSFileSystemStore.java | 16 +++
12+
.../hadoop/fs/aliyun/oss/Constants.java | 15 +++
13+
.../fs/aliyun/oss/ITAliyunOSSSignatureV4.java | 98 +++++++++++++++++++
14+
.../src/test/resources/log4j.properties | 3 +
15+
6 files changed, 159 insertions(+), 1 deletion(-)
16+
create mode 100644 hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
17+
18+
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
19+
index 3426a76c12..3a812ebc64 100644
20+
--- a/hadoop-project/pom.xml
21+
+++ b/hadoop-project/pom.xml
22+
@@ -1620,7 +1620,7 @@
23+
<dependency>
24+
<groupId>com.aliyun.oss</groupId>
25+
<artifactId>aliyun-sdk-oss</artifactId>
26+
- <version>3.13.2</version>
27+
+ <version>3.18.1</version>
28+
<exclusions>
29+
<exclusion>
30+
<groupId>org.apache.httpcomponents</groupId>
31+
diff --git a/hadoop-tools/hadoop-aliyun/pom.xml b/hadoop-tools/hadoop-aliyun/pom.xml
32+
index 5678cff642..3db04ce913 100644
33+
--- a/hadoop-tools/hadoop-aliyun/pom.xml
34+
+++ b/hadoop-tools/hadoop-aliyun/pom.xml
35+
@@ -165,5 +165,31 @@
36+
<scope>test</scope>
37+
<type>jar</type>
38+
</dependency>
39+
+
40+
+ <dependency>
41+
+ <groupId>org.junit.jupiter</groupId>
42+
+ <artifactId>junit-jupiter-api</artifactId>
43+
+ <scope>test</scope>
44+
+ </dependency>
45+
+ <dependency>
46+
+ <groupId>org.junit.jupiter</groupId>
47+
+ <artifactId>junit-jupiter-engine</artifactId>
48+
+ <scope>test</scope>
49+
+ </dependency>
50+
+ <dependency>
51+
+ <groupId>org.junit.jupiter</groupId>
52+
+ <artifactId>junit-jupiter-params</artifactId>
53+
+ <scope>test</scope>
54+
+ </dependency>
55+
+ <dependency>
56+
+ <groupId>org.junit.platform</groupId>
57+
+ <artifactId>junit-platform-launcher</artifactId>
58+
+ <scope>test</scope>
59+
+ </dependency>
60+
+ <dependency>
61+
+ <groupId>org.junit.vintage</groupId>
62+
+ <artifactId>junit-vintage-engine</artifactId>
63+
+ <scope>test</scope>
64+
+ </dependency>
65+
</dependencies>
66+
</project>
67+
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
68+
index 6e0c7dc7e4..dba267b04c 100644
69+
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
70+
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
71+
@@ -73,6 +73,7 @@ import java.util.List;
72+
import java.util.ListIterator;
73+
import java.util.NoSuchElementException;
74+
import java.util.stream.Collectors;
75+
+import com.aliyun.oss.common.comm.SignVersion;
76+
77+
import static org.apache.hadoop.fs.aliyun.oss.Constants.*;
78+
79+
@@ -113,6 +114,16 @@ public class AliyunOSSFileSystemStore {
80+
conf.get(USER_AGENT_PREFIX, USER_AGENT_PREFIX_DEFAULT) + ", Hadoop/"
81+
+ VersionInfo.getVersion());
82+
83+
+ String region = conf.get(REGION_KEY, "");
84+
+ String signatureVersion = conf.get(SIGNATURE_VERSION_KEY, SIGNATURE_VERSION_DEFAULT);
85+
+ if ("V4".equalsIgnoreCase(signatureVersion)) {
86+
+ clientConf.setSignatureVersion(SignVersion.V4);
87+
+ if (StringUtils.isEmpty(region)) {
88+
+ LOG.error("Signature version is V4 ,but region is empty.");
89+
+ throw new IOException("SignVersion is V4 but region is empty");
90+
+ }
91+
+ }
92+
+
93+
String proxyHost = conf.getTrimmed(PROXY_HOST_KEY, "");
94+
int proxyPort = conf.getInt(PROXY_PORT_KEY, -1);
95+
if (StringUtils.isNotEmpty(proxyHost)) {
96+
@@ -171,6 +182,11 @@ public class AliyunOSSFileSystemStore {
97+
statistics.incrementWriteOps(1);
98+
}
99+
100+
+ if (StringUtils.isNotEmpty(region)) {
101+
+ ossClient.setRegion(region);
102+
+ LOG.debug("ossClient setRegion {}", region);
103+
+ }
104+
+
105+
maxKeys = conf.getInt(MAX_PAGING_KEYS_KEY, MAX_PAGING_KEYS_DEFAULT);
106+
int listVersion = conf.getInt(LIST_VERSION, DEFAULT_LIST_VERSION);
107+
if (listVersion < 1 || listVersion > 2) {
108+
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
109+
index baeb919937..176669ed15 100644
110+
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
111+
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/Constants.java
112+
@@ -211,4 +211,19 @@ public final class Constants {
113+
public static final String LIST_VERSION = "fs.oss.list.version";
114+
115+
public static final int DEFAULT_LIST_VERSION = 2;
116+
+
117+
+ /**
118+
+ * OSS signature version.
119+
+ */
120+
+ public static final String SIGNATURE_VERSION_KEY = "fs.oss.signatureversion";
121+
+
122+
+ /**
123+
+ * OSS signature version DEFAULT {@value}.
124+
+ */
125+
+ public static final String SIGNATURE_VERSION_DEFAULT = "V1";
126+
+
127+
+ /**
128+
+ * OSS region {@value}.
129+
+ */
130+
+ public static final String REGION_KEY = "fs.oss.region";
131+
}
132+
diff --git a/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
133+
new file mode 100644
134+
index 0000000000..5070f2a581
135+
--- /dev/null
136+
+++ b/hadoop-tools/hadoop-aliyun/src/test/java/org/apache/hadoop/fs/aliyun/oss/ITAliyunOSSSignatureV4.java
137+
@@ -0,0 +1,98 @@
138+
+/**
139+
+ * Licensed to the Apache Software Foundation (ASF) under one
140+
+ * or more contributor license agreements. See the NOTICE file
141+
+ * distributed with this work for additional information
142+
+ * regarding copyright ownership. The ASF licenses this file
143+
+ * to you under the Apache License, Version 2.0 (the
144+
+ * "License"); you may not use this file except in compliance
145+
+ * with the License. You may obtain a copy of the License at
146+
+ *
147+
+ * http://www.apache.org/licenses/LICENSE-2.0
148+
+ *
149+
+ * Unless required by applicable law or agreed to in writing, software
150+
+ * distributed under the License is distributed on an "AS IS" BASIS,
151+
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
152+
+ * See the License for the specific language governing permissions and
153+
+ * limitations under the License.
154+
+ */
155+
+
156+
+package org.apache.hadoop.fs.aliyun.oss;
157+
+
158+
+import org.apache.hadoop.conf.Configuration;
159+
+import org.apache.hadoop.fs.FileStatus;
160+
+import org.apache.hadoop.fs.Path;
161+
+import org.junit.Before;
162+
+import org.junit.Test;
163+
+import org.slf4j.Logger;
164+
+import org.slf4j.LoggerFactory;
165+
+
166+
+import java.io.IOException;
167+
+import java.net.URI;
168+
+
169+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.REGION_KEY;
170+
+import static org.apache.hadoop.fs.aliyun.oss.Constants.SIGNATURE_VERSION_KEY;
171+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.createFile;
172+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.dataset;
173+
+import static org.junit.Assert.*;
174+
+import static org.junit.Assume.assumeNotNull;
175+
+
176+
+/**
177+
+ * Tests Aliyun OSS system.
178+
+ */
179+
+public class ITAliyunOSSSignatureV4 {
180+
+ private static final Logger LOG = LoggerFactory.getLogger(ITAliyunOSSSignatureV4.class);
181+
+ private Configuration conf;
182+
+ private URI testURI;
183+
+ private Path testFile = new Path("ITAliyunOSSSignatureV4/atestr");
184+
+
185+
+ @Before
186+
+ public void setUp() throws Exception {
187+
+ conf = new Configuration();
188+
+ String bucketUri = conf.get("test.fs.oss.name");
189+
+ LOG.debug("bucketUri={}", bucketUri);
190+
+ testURI = URI.create(bucketUri);
191+
+ }
192+
+
193+
+ @Test
194+
+ public void testV4() throws IOException {
195+
+ conf.set(SIGNATURE_VERSION_KEY, "V4");
196+
+ conf.set(REGION_KEY, "cn-hongkong");
197+
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
198+
+ fs.initialize(testURI, conf);
199+
+ assumeNotNull(fs);
200+
+
201+
+ createFile(fs, testFile, true, dataset(256, 0, 255));
202+
+ FileStatus status = fs.getFileStatus(testFile);
203+
+ fs.delete(testFile);
204+
+ fs.close();
205+
+ }
206+
+
207+
+ @Test
208+
+ public void testDefaultSignatureVersion() throws IOException {
209+
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
210+
+ fs.initialize(testURI, conf);
211+
+ assumeNotNull(fs);
212+
+
213+
+ Path testFile2 = new Path("/test/atestr");
214+
+ createFile(fs, testFile2, true, dataset(256, 0, 255));
215+
+ FileStatus status = fs.getFileStatus(testFile2);
216+
+ fs.delete(testFile2);
217+
+ fs.close();
218+
+ }
219+
+
220+
+ @Test
221+
+ public void testV4WithoutRegion() throws IOException {
222+
+ conf.set(SIGNATURE_VERSION_KEY, "V4");
223+
+ AliyunOSSFileSystem fs = new AliyunOSSFileSystem();
224+
+ IOException expectedException = null;
225+
+ try {
226+
+ fs.initialize(testURI, conf);
227+
+ } catch (IOException e) {
228+
+ LOG.warn("use V4 , but do not set region, get exception={}", e);
229+
+ expectedException = e;
230+
+ assertEquals("use V4 , but do not set region", e.getMessage(),
231+
+ "SignVersion is V4 but region is empty");
232+
+ }
233+
+ assertNotNull(expectedException);
234+
+ }
235+
+}
236+
diff --git a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
237+
index bb5cbe5ec3..2167f68811 100644
238+
--- a/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
239+
+++ b/hadoop-tools/hadoop-aliyun/src/test/resources/log4j.properties
240+
@@ -21,3 +21,6 @@ log4j.threshold=ALL
241+
log4j.appender.stdout=org.apache.log4j.ConsoleAppender
242+
log4j.appender.stdout.layout=org.apache.log4j.PatternLayout
243+
log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n
244+
+
245+
+# Log all oss classes
246+
+log4j.logger.org.apache.hadoop.fs.aliyun.oss=DEBUG
247+
\ No newline at end of file

0 commit comments

Comments
 (0)