Skip to content

Commit e1151e8

Browse files
XuQianJin-StarsXuQianJin-Stars
committed
[FLUSS-2686] Add COS filesystem support
This adds a new fluss-fs-cos module that integrates Tencent Cloud COS (Cloud Object Storage) as a remote filesystem for Fluss. Key changes: - Add fluss-fs-cos module with hadoop-cos and cos_api dependencies - Implement COSFileSystem extending HadoopFileSystem (scheme: cosn) - Implement COSFileSystemPlugin as FileSystemPlugin SPI - Add security token support (COSSecurityTokenProvider/Receiver) - Add DynamicTemporaryCOSCredentialsProvider for temporary credentials - Add integration tests for COS filesystem behavior - Register module in fluss-filesystems parent pom
1 parent dd181eb commit e1151e8

File tree

16 files changed

+1196
-0
lines changed

16 files changed

+1196
-0
lines changed

‎fluss-filesystems/fluss-fs-cos/pom.xml‎

Lines changed: 254 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,254 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
20+
<project xmlns="http://maven.apache.org/POM/4.0.0"
21+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
22+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
23+
<modelVersion>4.0.0</modelVersion>
24+
<parent>
25+
<groupId>org.apache.fluss</groupId>
26+
<artifactId>fluss-filesystems</artifactId>
27+
<version>0.10-SNAPSHOT</version>
28+
</parent>
29+
30+
<artifactId>fluss-fs-cos</artifactId>
31+
<name>Fluss : FileSystems : COS FS</name>
32+
33+
<properties>
34+
<fs.cosn.sdk.version>3.3.5</fs.cosn.sdk.version>
35+
<fs.cos.api.version>5.6.139</fs.cos.api.version>
36+
</properties>
37+
38+
<dependencies>
39+
<dependency>
40+
<groupId>org.apache.fluss</groupId>
41+
<artifactId>fluss-common</artifactId>
42+
<version>${project.version}</version>
43+
<scope>provided</scope>
44+
</dependency>
45+
46+
<dependency>
47+
<groupId>org.apache.fluss</groupId>
48+
<artifactId>fluss-fs-hadoop-shaded</artifactId>
49+
<version>${project.version}</version>
50+
</dependency>
51+
52+
<dependency>
53+
<groupId>org.apache.fluss</groupId>
54+
<artifactId>fluss-fs-hadoop</artifactId>
55+
<version>${project.version}</version>
56+
</dependency>
57+
58+
<dependency>
59+
<groupId>org.apache.hadoop</groupId>
60+
<artifactId>hadoop-cos</artifactId>
61+
<version>${fs.cosn.sdk.version}</version>
62+
<exclusions>
63+
<exclusion>
64+
<groupId>com.qcloud</groupId>
65+
<artifactId>cos_api</artifactId>
66+
</exclusion>
67+
<exclusion>
68+
<groupId>org.apache.hadoop</groupId>
69+
<artifactId>hadoop-common</artifactId>
70+
</exclusion>
71+
<exclusion>
72+
<groupId>ch.qos.reload4j</groupId>
73+
<artifactId>reload4j</artifactId>
74+
</exclusion>
75+
<exclusion>
76+
<groupId>org.slf4j</groupId>
77+
<artifactId>slf4j-reload4j</artifactId>
78+
</exclusion>
79+
</exclusions>
80+
</dependency>
81+
82+
<dependency>
83+
<groupId>com.qcloud</groupId>
84+
<artifactId>cos_api</artifactId>
85+
<version>${fs.cos.api.version}</version>
86+
<exclusions>
87+
<exclusion>
88+
<groupId>javax.xml.bind</groupId>
89+
<artifactId>jaxb-api</artifactId>
90+
</exclusion>
91+
</exclusions>
92+
</dependency>
93+
94+
<dependency>
95+
<!-- Hadoop requires jaxb-api for javax.xml.bind.JAXBException -->
96+
<groupId>javax.xml.bind</groupId>
97+
<artifactId>jaxb-api</artifactId>
98+
<version>${jaxb.api.version}</version>
99+
<!-- packaged as an optional dependency that is only accessible on Java 11+ -->
100+
<scope>provided</scope>
101+
</dependency>
102+
103+
<dependency>
104+
<groupId>org.apache.fluss</groupId>
105+
<artifactId>fluss-test-utils</artifactId>
106+
</dependency>
107+
<!-- for the behavior test suite -->
108+
<dependency>
109+
<groupId>org.apache.fluss</groupId>
110+
<artifactId>fluss-common</artifactId>
111+
<version>${project.version}</version>
112+
<scope>test</scope>
113+
<type>test-jar</type>
114+
</dependency>
115+
</dependencies>
116+
117+
<build>
118+
<plugins>
119+
<plugin>
120+
<groupId>org.apache.maven.plugins</groupId>
121+
<artifactId>maven-jar-plugin</artifactId>
122+
<executions>
123+
<execution>
124+
<goals>
125+
<goal>jar</goal>
126+
</goals>
127+
</execution>
128+
</executions>
129+
<configuration>
130+
<archive>
131+
<manifestEntries>
132+
<!-- jaxb-api is packaged as an optional dependency that is only accessible on Java 11 -->
133+
<Multi-Release>true</Multi-Release>
134+
</manifestEntries>
135+
</archive>
136+
</configuration>
137+
</plugin>
138+
139+
<plugin>
140+
<groupId>org.apache.maven.plugins</groupId>
141+
<artifactId>maven-dependency-plugin</artifactId>
142+
<executions>
143+
<execution>
144+
<id>copy-javax-jars</id>
145+
<phase>process-resources</phase>
146+
<goals>
147+
<goal>copy</goal>
148+
</goals>
149+
</execution>
150+
</executions>
151+
<configuration>
152+
<artifactItems>
153+
<artifactItem>
154+
<groupId>javax.xml.bind</groupId>
155+
<artifactId>jaxb-api</artifactId>
156+
<version>${jaxb.api.version}</version>
157+
<type>jar</type>
158+
<overWrite>true</overWrite>
159+
</artifactItem>
160+
</artifactItems>
161+
<outputDirectory>${project.build.directory}/temporary</outputDirectory>
162+
</configuration>
163+
</plugin>
164+
165+
<plugin>
166+
<groupId>org.apache.maven.plugins</groupId>
167+
<artifactId>maven-antrun-plugin</artifactId>
168+
<executions>
169+
<execution>
170+
<id>unpack-javax-libraries</id>
171+
<phase>process-resources</phase>
172+
<goals>
173+
<goal>run</goal>
174+
</goals>
175+
<configuration>
176+
<target>
177+
<echo message="unpacking javax jars"/>
178+
<unzip dest="${project.build.directory}/classes/META-INF/versions/11">
179+
<fileset dir="${project.build.directory}/temporary">
180+
<include name="*"/>
181+
</fileset>
182+
</unzip>
183+
</target>
184+
</configuration>
185+
</execution>
186+
</executions>
187+
</plugin>
188+
189+
<plugin>
190+
<groupId>org.apache.maven.plugins</groupId>
191+
<artifactId>maven-shade-plugin</artifactId>
192+
<executions>
193+
<execution>
194+
<id>shade-fluss</id>
195+
<phase>package</phase>
196+
<goals>
197+
<goal>shade</goal>
198+
</goals>
199+
<configuration>
200+
<artifactSet>
201+
<includes>
202+
<include>*:*</include>
203+
</includes>
204+
<excludes>
205+
<exclude>javax.servlet:servlet-api</exclude>
206+
<exclude>xmlenc:xmlenc</exclude>
207+
</excludes>
208+
</artifactSet>
209+
<filters>
210+
<filter>
211+
<artifact>*</artifact>
212+
<excludes>
213+
<exclude>.gitkeep</exclude>
214+
<exclude>mime.types</exclude>
215+
<exclude>mozilla/**</exclude>
216+
<exclude>LICENSE.txt</exclude>
217+
<exclude>license/LICENSE*</exclude>
218+
<exclude>okhttp3/internal/publicsuffix/NOTICE</exclude>
219+
<exclude>NOTICE</exclude>
220+
</excludes>
221+
</filter>
222+
<filter>
223+
<artifact>org.apache.fluss:fluss-fs-hadoop</artifact>
224+
<excludes>
225+
<exclude>META-INF/**</exclude>
226+
</excludes>
227+
</filter>
228+
</filters>
229+
<relocations>
230+
<relocation>
231+
<pattern>org.apache.commons</pattern>
232+
<shadedPattern>org.apache.fluss.shaded.org.apache.commons</shadedPattern>
233+
</relocation>
234+
</relocations>
235+
</configuration>
236+
</execution>
237+
</executions>
238+
</plugin>
239+
240+
<plugin>
241+
<groupId>org.apache.maven.plugins</groupId>
242+
<artifactId>maven-jar-plugin</artifactId>
243+
<executions>
244+
<execution>
245+
<goals>
246+
<goal>test-jar</goal>
247+
</goals>
248+
</execution>
249+
</executions>
250+
</plugin>
251+
</plugins>
252+
</build>
253+
254+
</project>

‎fluss-filesystems/fluss-fs-cos/src/main/java/org/apache/fluss/fs/cos/COSFileSystem.java‎

Lines changed: 64 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,64 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*/
17+
18+
package org.apache.fluss.fs.cos;
19+
20+
import org.apache.fluss.fs.cos.token.COSSecurityTokenProvider;
21+
import org.apache.fluss.fs.hdfs.HadoopFileSystem;
22+
import org.apache.fluss.fs.token.ObtainedSecurityToken;
23+
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.apache.hadoop.fs.FileSystem;
26+
27+
import java.io.IOException;
28+
29+
/**
30+
* A {@link FileSystem} for Tencent Cloud COS that wraps an {@link HadoopFileSystem}, but overwrite
31+
* method to generate access security token.
32+
*/
33+
class COSFileSystem extends HadoopFileSystem {
34+
35+
private final Configuration conf;
36+
private volatile COSSecurityTokenProvider cosSecurityTokenProvider;
37+
private final String scheme;
38+
39+
COSFileSystem(FileSystem hadoopFileSystem, String scheme, Configuration conf) {
40+
super(hadoopFileSystem);
41+
this.scheme = scheme;
42+
this.conf = conf;
43+
}
44+
45+
@Override
46+
public ObtainedSecurityToken obtainSecurityToken() throws IOException {
47+
try {
48+
mayCreateSecurityTokenProvider();
49+
return cosSecurityTokenProvider.obtainSecurityToken(scheme);
50+
} catch (Exception e) {
51+
throw new IOException(e);
52+
}
53+
}
54+
55+
private void mayCreateSecurityTokenProvider() throws IOException {
56+
if (cosSecurityTokenProvider == null) {
57+
synchronized (this) {
58+
if (cosSecurityTokenProvider == null) {
59+
cosSecurityTokenProvider = new COSSecurityTokenProvider(conf);
60+
}
61+
}
62+
}
63+
}
64+
}

0 commit comments

Comments
 (0)