Skip to content

Commit aa75250

Browse files
yuanoOoyuanoOo
authored
feat: add flink-sql-connector-obkv-hbase-shade module (#130)
1 parent 8f751cf commit aa75250

File tree

3 files changed

+243
-0
lines changed

3 files changed

+243
-0
lines changed

‎flink-sql-connector-obkv-hbase-shade/pom.xml‎

Lines changed: 203 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,203 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Copyright 2024 OceanBase.
4+
5+
Licensed under the Apache License, Version 2.0 (the "License");
6+
you may not use this file except in compliance with the License.
7+
You may obtain a copy of the License at
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
Unless required by applicable law or agreed to in writing,
10+
software distributed under the License is distributed on an
11+
"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
12+
KIND, either express or implied. See the License for the
13+
specific language governing permissions and limitations
14+
under the License.
15+
-->
16+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
17+
<modelVersion>4.0.0</modelVersion>
18+
<parent>
19+
<groupId>com.oceanbase</groupId>
20+
<artifactId>flink-connector-oceanbase-parent</artifactId>
21+
<version>${revision}</version>
22+
</parent>
23+
24+
<artifactId>flink-sql-connector-obkv-hbase-shade</artifactId>
25+
26+
<dependencies>
27+
<dependency>
28+
<groupId>com.oceanbase</groupId>
29+
<artifactId>obkv-hbase-client</artifactId>
30+
<version>2.4.0</version>
31+
</dependency>
32+
<!-- https://mvnrepository.com/artifact/org.apache.flink/flink-connector-hbase-2.2 -->
33+
<dependency>
34+
<groupId>org.apache.flink</groupId>
35+
<artifactId>flink-connector-hbase-2.2</artifactId>
36+
<version>4.0.0-1.19</version>
37+
</dependency>
38+
</dependencies>
39+
<build>
40+
<plugins>
41+
<plugin>
42+
<groupId>org.apache.maven.plugins</groupId>
43+
<artifactId>maven-jar-plugin</artifactId>
44+
<executions>
45+
<execution>
46+
<id>empty-javadoc-jar</id>
47+
<goals>
48+
<goal>jar</goal>
49+
</goals>
50+
<phase>package</phase>
51+
<configuration>
52+
<classifier>javadoc</classifier>
53+
<classesDirectory>${basedir}/src/main/java</classesDirectory>
54+
</configuration>
55+
</execution>
56+
</executions>
57+
</plugin>
58+
<plugin>
59+
<groupId>org.apache.maven.plugins</groupId>
60+
<artifactId>maven-shade-plugin</artifactId>
61+
<version>3.5.3</version>
62+
<executions>
63+
<execution>
64+
<id>shade-flink</id>
65+
<goals>
66+
<goal>shade</goal>
67+
</goals>
68+
<phase>package</phase>
69+
<configuration>
70+
<shadeTestJar>false</shadeTestJar>
71+
<createDependencyReducedPom>false</createDependencyReducedPom>
72+
<shadedArtifactAttached>false</shadedArtifactAttached>
73+
<artifactSet>
74+
<includes>
75+
<!-- Include OceanBase related clients -->
76+
<include>com.oceanbase:*</include>
77+
<!-- Include Alipay related dependencies -->
78+
<include>com.alipay.*:*</include>
79+
<!-- Include Alibaba dependencies -->
80+
<include>com.alibaba:*</include>
81+
<!-- Include Google related libraries -->
82+
<include>com.google.*:*</include>
83+
<!-- Include Jackson libraries -->
84+
<include>com.fasterxml.jackson.core:jackson-core</include>
85+
<include>com.fasterxml.jackson.core:jackson-databind</include>
86+
<include>com.fasterxml.jackson.core:jackson-annotations</include>
87+
<include>com.fasterxml.jackson.dataformat:*</include>
88+
<!-- Include Netty networking library -->
89+
<include>io.netty:*</include>
90+
<!-- Include Hadoop related -->
91+
<include>org.apache.hadoop:*</include>
92+
<!-- Include HBase client -->
93+
<include>org.apache.hbase:*</include>
94+
<include>org.apache.hbase.thirdparty:*</include>
95+
<!-- Include Flink HBase connectors -->
96+
<include>org.apache.flink:flink-connector-hbase-2.2</include>
97+
<include>org.apache.flink:flink-connector-hbase-base</include>
98+
<!-- Include Apache Commons libraries (Hadoop dependencies) -->
99+
<include>commons-logging:commons-logging</include>
100+
<include>commons-lang:commons-lang</include>
101+
<include>org.apache.commons:commons-lang3</include>
102+
<include>commons-configuration:commons-configuration</include>
103+
<include>commons-cli:commons-cli</include>
104+
<include>commons-collections:commons-collections</include>
105+
<include>commons-io:commons-io</include>
106+
<include>commons-codec:commons-codec</include>
107+
</includes>
108+
</artifactSet>
109+
<relocations>
110+
<!-- Shade Log4j to avoid conflicts -->
111+
<relocation>
112+
<pattern>org.apache.log4j</pattern>
113+
<shadedPattern>${shaded.prefix}.org.apache.log4j</shadedPattern>
114+
</relocation>
115+
<!-- Shade Log4j2 to avoid conflicts with Flink's Log4j2 for obkv-table-client -->
116+
<relocation>
117+
<pattern>org.apache.logging.log4j</pattern>
118+
<shadedPattern>${shaded.prefix}.org.apache.logging.log4j</shadedPattern>
119+
</relocation>
120+
<relocation>
121+
<pattern>org.apache.logging.slf4j</pattern>
122+
<shadedPattern>${shaded.prefix}.org.apache.logging.slf4j</shadedPattern>
123+
</relocation>
124+
<!-- Shade Apache Commons libraries to match shaded Hadoop -->
125+
<relocation>
126+
<pattern>org.apache.commons.logging</pattern>
127+
<shadedPattern>${shaded.prefix}.org.apache.commons.logging</shadedPattern>
128+
</relocation>
129+
<relocation>
130+
<pattern>org.apache.commons.lang</pattern>
131+
<shadedPattern>${shaded.prefix}.org.apache.commons.lang</shadedPattern>
132+
</relocation>
133+
<relocation>
134+
<pattern>org.apache.commons.lang3</pattern>
135+
<shadedPattern>${shaded.prefix}.org.apache.commons.lang3</shadedPattern>
136+
</relocation>
137+
<relocation>
138+
<pattern>org.apache.commons.configuration</pattern>
139+
<shadedPattern>${shaded.prefix}.org.apache.commons.configuration</shadedPattern>
140+
</relocation>
141+
<relocation>
142+
<pattern>org.apache.commons.cli</pattern>
143+
<shadedPattern>${shaded.prefix}.org.apache.commons.cli</shadedPattern>
144+
</relocation>
145+
<relocation>
146+
<pattern>org.apache.commons.collections</pattern>
147+
<shadedPattern>${shaded.prefix}.org.apache.commons.collections</shadedPattern>
148+
</relocation>
149+
<relocation>
150+
<pattern>org.apache.commons.io</pattern>
151+
<shadedPattern>${shaded.prefix}.org.apache.commons.io</shadedPattern>
152+
</relocation>
153+
<relocation>
154+
<pattern>org.apache.commons.codec</pattern>
155+
<shadedPattern>${shaded.prefix}.org.apache.commons.codec</shadedPattern>
156+
</relocation>
157+
<!-- Shade Jackson to avoid version conflicts -->
158+
<relocation>
159+
<pattern>com.fasterxml.jackson</pattern>
160+
<shadedPattern>${shaded.prefix}.com.fasterxml.jackson</shadedPattern>
161+
</relocation>
162+
<!-- Shade Guava to avoid version conflicts -->
163+
<relocation>
164+
<pattern>com.google</pattern>
165+
<shadedPattern>${shaded.prefix}.com.google</shadedPattern>
166+
</relocation>
167+
<!-- Shade Hadoop classes -->
168+
<relocation>
169+
<pattern>org.apache.hadoop</pattern>
170+
<shadedPattern>${shaded.prefix}.org.apache.hadoop</shadedPattern>
171+
</relocation>
172+
<!-- Shade Netty to avoid conflicts -->
173+
<relocation>
174+
<pattern>io.netty</pattern>
175+
<shadedPattern>${shaded.prefix}.io.netty</shadedPattern>
176+
</relocation>
177+
</relocations>
178+
<filters>
179+
<filter>
180+
<artifact>*:*</artifact>
181+
<excludes>
182+
<exclude>META-INF/*.SF</exclude>
183+
<exclude>META-INF/*.DSA</exclude>
184+
<exclude>META-INF/*.RSA</exclude>
185+
<exclude>META-INF/NOTICE</exclude>
186+
<exclude>META-INF/NOTICE.txt</exclude>
187+
<exclude>META-INF/LICENSE</exclude>
188+
<exclude>META-INF/LICENSE.txt</exclude>
189+
<exclude>META-INF/DEPENDENCIES</exclude>
190+
</excludes>
191+
</filter>
192+
</filters>
193+
<transformers>
194+
<!-- Process META-INF/services files -->
195+
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
196+
</transformers>
197+
</configuration>
198+
</execution>
199+
</executions>
200+
</plugin>
201+
</plugins>
202+
</build>
203+
</project>

‎flink-sql-connector-obkv-hbase-shade/src/main/resources/core-site.xml‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
3+
<!--
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License. See accompanying LICENSE file.
15+
-->
16+
17+
<!-- Hadoop core-site.xml override for shaded connector -->
18+
<configuration>
19+
<!--
20+
Override Hadoop default groups mapping implementation
21+
Use ShellBasedUnixGroupsMapping instead of JniBasedUnixGroupsMappingWithFallback
22+
To avoid class loading issues after shading
23+
-->
24+
<property>
25+
<name>hadoop.security.group.mapping</name>
26+
<value>com.oceanbase.connector.flink.shaded.org.apache.hadoop.security.ShellBasedUnixGroupsMapping</value>
27+
<description>
28+
Class for user to group mapping (get groups for a given user) for ACL.
29+
The default implementation uses JNI which may cause issues when shaded.
30+
This override uses the shell-based implementation instead.
31+
</description>
32+
</property>
33+
34+
<!-- Disable Hadoop metrics system to reduce dependencies and potential issues -->
35+
<property>
36+
<name>hadoop.security.instrumentation.requires.admin</name>
37+
<value>false</value>
38+
</property>
39+
</configuration>

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ under the License.
4040
<module>flink-sql-connector-oceanbase-directload</module>
4141
<module>flink-connector-oceanbase-cli</module>
4242
<module>flink-connector-oceanbase-e2e-tests</module>
43+
<module>flink-sql-connector-obkv-hbase-shade</module>
4344
</modules>
4445

4546
<properties>

0 commit comments

Comments
 (0)