Skip to content

Commit 39adc59

Browse files
andybj0228andybj0228
committed
[File system] Add HuaweiCloud OBS integration
1 parent 98c2f47 commit 39adc59

File tree

22 files changed

+1415
-0
lines changed

22 files changed

+1415
-0
lines changed

‎fluss-dist/pom.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -68,6 +68,13 @@
6868
<scope>provided</scope>
6969
</dependency>
7070

71+
<dependency>
72+
<groupId>com.alibaba.fluss</groupId>
73+
<artifactId>fluss-fs-obs</artifactId>
74+
<version>${project.version}</version>
75+
<scope>provided</scope>
76+
</dependency>
77+
7178
<!-- metrics plugin -->
7279
<dependency>
7380
<groupId>com.alibaba.fluss</groupId>

‎fluss-dist/src/main/assemblies/plugins.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,12 @@
4545
<files>
4646
<!-- filesystem -->
4747
<!-- output directory should correspond to the file system *schema* name, i.e., plugins/<schema>/ -->
48+
<file>
49+
<source>../fluss-filesystems/fluss-fs-obs/target/fluss-fs-obs-${project.version}.jar</source>
50+
<outputDirectory>plugins/obs/</outputDirectory>
51+
<destName>fluss-fs-obs-${project.version}.jar</destName>
52+
<fileMode>0644</fileMode>
53+
</file>
4854
<file>
4955
<source>../fluss-filesystems/fluss-fs-gs/target/fluss-fs-gs-${project.version}.jar</source>
5056
<outputDirectory>plugins/gs/</outputDirectory>

‎fluss-filesystems/fluss-fs-obs/pom.xml‎

Lines changed: 261 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,261 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<groupId>com.alibaba.fluss</groupId>
26+
<artifactId>fluss-filesystems</artifactId>
27+
<version>0.8-SNAPSHOT</version>
28+
</parent>
29+
30+
<artifactId>fluss-fs-obs</artifactId>
31+
<name>Fluss : FileSystems : OBS FS</name>
32+
33+
<properties>
34+
<fs.obs.sdk.version>3.24.12</fs.obs.sdk.version>
35+
<huaweicloud.sdk.iam.version>3.1.87</huaweicloud.sdk.iam.version>
36+
<hadoop.huaweicloud.version>3.4.0</hadoop.huaweicloud.version>
37+
</properties>
38+
39+
<dependencies>
40+
<dependency>
41+
<groupId>com.alibaba.fluss</groupId>
42+
<artifactId>fluss-common</artifactId>
43+
<version>${project.version}</version>
44+
<scope>provided</scope>
45+
</dependency>
46+
47+
<dependency>
48+
<groupId>com.alibaba.fluss</groupId>
49+
<artifactId>fluss-fs-hadoop-shaded</artifactId>
50+
<version>${project.version}</version>
51+
</dependency>
52+
53+
<dependency>
54+
<groupId>com.alibaba.fluss</groupId>
55+
<artifactId>fluss-fs-hadoop</artifactId>
56+
<version>${project.version}</version>
57+
<exclusions>
58+
<exclusion>
59+
<groupId>org.apache.hadoop</groupId>
60+
<artifactId>hadoop-hdfs-client</artifactId>
61+
</exclusion>
62+
</exclusions>
63+
</dependency>
64+
65+
<dependency>
66+
<groupId>org.apache.hadoop</groupId>
67+
<artifactId>hadoop-huaweicloud</artifactId>
68+
<version>${hadoop.huaweicloud.version}</version>
69+
<exclusions>
70+
<exclusion>
71+
<groupId>com.huaweicloud</groupId>
72+
<artifactId>esdk-obs-java-bundle</artifactId>
73+
</exclusion>
74+
<exclusion>
75+
<groupId>org.apache.hadoop</groupId>
76+
<artifactId>hadoop-common</artifactId>
77+
</exclusion>
78+
<exclusion>
79+
<groupId>ch.qos.reload4j</groupId>
80+
<artifactId>reload4j</artifactId>
81+
</exclusion>
82+
<exclusion>
83+
<groupId>org.slf4j</groupId>
84+
<artifactId>slf4j-reload4j</artifactId>
85+
</exclusion>
86+
</exclusions>
87+
</dependency>
88+
89+
<dependency>
90+
<groupId>com.huaweicloud</groupId>
91+
<artifactId>esdk-obs-java</artifactId>
92+
<version>${fs.obs.sdk.version}</version>
93+
<exclusions>
94+
<exclusion>
95+
<groupId>javax.xml.bind</groupId>
96+
<artifactId>jaxb-api</artifactId>
97+
</exclusion>
98+
</exclusions>
99+
</dependency>
100+
101+
<dependency>
102+
<groupId>com.huaweicloud.sdk</groupId>
103+
<artifactId>huaweicloud-sdk-iam</artifactId>
104+
<version>${huaweicloud.sdk.iam.version}</version>
105+
</dependency>
106+
107+
<dependency>
108+
<!-- Hadoop requires jaxb-api for javax.xml.bind.JAXBException -->
109+
<groupId>javax.xml.bind</groupId>
110+
<artifactId>jaxb-api</artifactId>
111+
<version>${jaxb.api.version}</version>
112+
<!-- packaged as an optional dependency that is only accessible on Java 11+ -->
113+
<scope>provided</scope>
114+
</dependency>
115+
116+
<dependency>
117+
<groupId>com.alibaba.fluss</groupId>
118+
<artifactId>fluss-test-utils</artifactId>
119+
</dependency>
120+
<!-- for the behavior test suite -->
121+
<dependency>
122+
<groupId>com.alibaba.fluss</groupId>
123+
<artifactId>fluss-common</artifactId>
124+
<version>${project.version}</version>
125+
<scope>test</scope>
126+
<type>test-jar</type>
127+
</dependency>
128+
</dependencies>
129+
130+
<build>
131+
<plugins>
132+
<plugin>
133+
<groupId>org.apache.maven.plugins</groupId>
134+
<artifactId>maven-jar-plugin</artifactId>
135+
<executions>
136+
<execution>
137+
<goals>
138+
<goal>jar</goal>
139+
</goals>
140+
</execution>
141+
</executions>
142+
<configuration>
143+
<archive>
144+
<manifestEntries>
145+
<!-- jaxb-api is packaged as an optional dependency that is only accessible on Java 11 -->
146+
<Multi-Release>true</Multi-Release>
147+
</manifestEntries>
148+
</archive>
149+
</configuration>
150+
</plugin>
151+
152+
<plugin>
153+
<groupId>org.apache.maven.plugins</groupId>
154+
<artifactId>maven-dependency-plugin</artifactId>
155+
<executions>
156+
<execution>
157+
<id>copy-javax-jars</id>
158+
<phase>process-resources</phase>
159+
<goals>
160+
<goal>copy</goal>
161+
</goals>
162+
</execution>
163+
</executions>
164+
<configuration>
165+
<artifactItems>
166+
<artifactItem>
167+
<groupId>javax.xml.bind</groupId>
168+
<artifactId>jaxb-api</artifactId>
169+
<version>${jaxb.api.version}</version>
170+
<type>jar</type>
171+
<overWrite>true</overWrite>
172+
</artifactItem>
173+
</artifactItems>
174+
<outputDirectory>${project.build.directory}/temporary</outputDirectory>
175+
</configuration>
176+
</plugin>
177+
178+
<plugin>
179+
<groupId>org.apache.maven.plugins</groupId>
180+
<artifactId>maven-antrun-plugin</artifactId>
181+
<executions>
182+
<execution>
183+
<id>unpack-javax-libraries</id>
184+
<phase>process-resources</phase>
185+
<goals>
186+
<goal>run</goal>
187+
</goals>
188+
<configuration>
189+
<target>
190+
<echo message="unpacking javax jars"/>
191+
<unzip dest="${project.build.directory}/classes/META-INF/versions/11">
192+
<fileset dir="${project.build.directory}/temporary">
193+
<include name="*"/>
194+
</fileset>
195+
</unzip>
196+
</target>
197+
</configuration>
198+
</execution>
199+
</executions>
200+
</plugin>
201+
202+
<plugin>
203+
<groupId>org.apache.maven.plugins</groupId>
204+
<artifactId>maven-shade-plugin</artifactId>
205+
<executions>
206+
<execution>
207+
<id>shade-fluss</id>
208+
<phase>package</phase>
209+
<goals>
210+
<goal>shade</goal>
211+
</goals>
212+
<configuration>
213+
<artifactSet>
214+
<includes>
215+
<include>*:*</include>
216+
</includes>
217+
<excludes>
218+
<exclude>javax.servlet:servlet-api</exclude>
219+
<exclude>xmlenc:xmlenc</exclude>
220+
</excludes>
221+
</artifactSet>
222+
<filters>
223+
<filter>
224+
<artifact>*</artifact>
225+
<excludes>
226+
<exclude>.gitkeep</exclude>
227+
<exclude>mime.types</exclude>
228+
<exclude>mozilla/**</exclude>
229+
<exclude>LICENSE.txt</exclude>
230+
<exclude>license/LICENSE*</exclude>
231+
<exclude>okhttp3/internal/publicsuffix/NOTICE</exclude>
232+
<exclude>NOTICE</exclude>
233+
</excludes>
234+
</filter>
235+
</filters>
236+
<relocations>
237+
<relocation>
238+
<pattern>org.apache.commons</pattern>
239+
<shadedPattern>com.alibaba.fluss.shaded.org.apache.commons</shadedPattern>
240+
</relocation>
241+
</relocations>
242+
</configuration>
243+
</execution>
244+
</executions>
245+
</plugin>
246+
247+
<plugin>
248+
<groupId>org.apache.maven.plugins</groupId>
249+
<artifactId>maven-jar-plugin</artifactId>
250+
<executions>
251+
<execution>
252+
<goals>
253+
<goal>test-jar</goal>
254+
</goals>
255+
</execution>
256+
</executions>
257+
</plugin>
258+
</plugins>
259+
</build>
260+
261+
</project>

‎fluss-filesystems/fluss-fs-obs/src/main/java/com/alibaba/fluss/fs/obs/OBSFileSystem.java‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package com.alibaba.fluss.fs.obs;
19+
20+
import com.alibaba.fluss.fs.hdfs.HadoopFileSystem;
21+
import com.alibaba.fluss.fs.obs.token.OBSSecurityTokenProvider;
22+
import com.alibaba.fluss.fs.token.ObtainedSecurityToken;
23+
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.apache.hadoop.fs.FileSystem;
26+
27+
import java.io.IOException;
28+
29+
/* This file is based on source code of Apache Flink Project (https://flink.apache.org/), licensed by the Apache
30+
* Software Foundation (ASF) under the Apache License, Version 2.0. See the NOTICE file distributed with this work for
31+
* additional information regarding copyright ownership. */
32+
33+
/**
34+
* A {@link FileSystem} for HuaweiCloud OBS that wraps an {@link HadoopFileSystem}, but overwrite
35+
* method to generate access security token.
36+
*/
37+
class OBSFileSystem extends HadoopFileSystem {
38+
39+
private final Configuration conf;
40+
private volatile OBSSecurityTokenProvider obsSecurityTokenProvider;
41+
private final String scheme;
42+
43+
OBSFileSystem(FileSystem hadoopFileSystem, String scheme, Configuration conf) {
44+
super(hadoopFileSystem);
45+
this.scheme = scheme;
46+
this.conf = conf;
47+
}
48+
49+
@Override
50+
public ObtainedSecurityToken obtainSecurityToken() throws IOException {
51+
try {
52+
mayCreateSecurityTokenProvider();
53+
return obsSecurityTokenProvider.obtainSecurityToken(scheme);
54+
} catch (Exception e) {
55+
throw new IOException(e);
56+
}
57+
}
58+
59+
private void mayCreateSecurityTokenProvider() throws IOException {
60+
if (obsSecurityTokenProvider == null) {
61+
synchronized (this) {
62+
if (obsSecurityTokenProvider == null) {
63+
obsSecurityTokenProvider = new OBSSecurityTokenProvider(conf);
64+
}
65+
}
66+
}
67+
}
68+
}

0 commit comments

Comments
 (0)