Skip to content

Commit 3c66d07

Browse files
dujieyanghuaiGit
dujie
authored and
yanghuaiGit
committed
[feat-882][inceptor]Use file reading instead of JDBC
1 parent 1a15c87 commit 3c66d07

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

41 files changed

+5753
-71
lines changed

‎chunjun-connectors/chunjun-connector-hbase-1.4/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,12 @@
4141
<groupId>com.dtstack.chunjun</groupId>
4242
<artifactId>chunjun-connector-hbase-base</artifactId>
4343
<version>${project.version}</version>
44+
<exclusions>
45+
<exclusion>
46+
<artifactId>log4j-1.2-api</artifactId>
47+
<groupId>org.apache.logging.log4j</groupId>
48+
</exclusion>
49+
</exclusions>
4450
</dependency>
4551
<dependency>
4652
<groupId>org.hbase</groupId>

‎chunjun-connectors/chunjun-connector-inceptor/pom.xml‎

Lines changed: 219 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -13,13 +13,158 @@
1313
<name>ChunJun : Connectors : Inceptor</name>
1414

1515
<dependencies>
16+
<dependency>
17+
<groupId>io.transwarp.wrdecimal</groupId>
18+
<artifactId>wrdecimal-library</artifactId>
19+
<version>0.1</version>
20+
</dependency>
21+
22+
<dependency>
23+
<artifactId>inceptor-serde</artifactId>
24+
<groupId>org.apache.hive</groupId>
25+
<version>8.0.1</version>
26+
</dependency>
27+
28+
29+
<dependency>
30+
<groupId>org.apache.hive</groupId>
31+
<artifactId>inceptor-exec</artifactId>
32+
<version>8.0.1</version>
33+
</dependency>
34+
35+
36+
<dependency>
37+
<groupId>org.apache.hive</groupId>
38+
<artifactId>inceptor-metastore</artifactId>
39+
<version>8.0.1</version>
40+
</dependency>
41+
42+
1643
<!-- 星环 inceptor 依赖 -->
1744
<dependency>
1845
<groupId>io.transwarp</groupId>
1946
<artifactId>inceptor-driver</artifactId>
2047
<version>6.0.2</version>
48+
<exclusions>
49+
<exclusion>
50+
<groupId>org.apache.hadoop</groupId>
51+
<artifactId>hadoop-hdfs</artifactId>
52+
</exclusion>
53+
<exclusion>
54+
<groupId>commons-cli</groupId>
55+
<artifactId>commons-cli</artifactId>
56+
</exclusion>
57+
</exclusions>
2158
</dependency>
2259

60+
<dependency>
61+
<groupId>commons-cli</groupId>
62+
<artifactId>commons-cli</artifactId>
63+
<version>1.3.1</version>
64+
</dependency>
65+
66+
<dependency>
67+
<artifactId>parquet-hadoop</artifactId>
68+
<groupId>org.apache.parquet</groupId>
69+
<version>1.8.3</version>
70+
<exclusions>
71+
<exclusion>
72+
<groupId>org.xerial.snappy</groupId>
73+
<artifactId>snappy-java</artifactId>
74+
</exclusion>
75+
<exclusion>
76+
<artifactId>commons-codec</artifactId>
77+
<groupId>commons-codec</groupId>
78+
</exclusion>
79+
<exclusion>
80+
<artifactId>jackson-core-asl</artifactId>
81+
<groupId>org.codehaus.jackson</groupId>
82+
</exclusion>
83+
<exclusion>
84+
<artifactId>jackson-mapper-asl</artifactId>
85+
<groupId>org.codehaus.jackson</groupId>
86+
</exclusion>
87+
<exclusion>
88+
<artifactId>slf4j-api</artifactId>
89+
<groupId>org.slf4j</groupId>
90+
</exclusion>
91+
</exclusions>
92+
</dependency>
93+
94+
<dependency>
95+
<groupId>org.apache.thrift</groupId>
96+
<artifactId>libfb303</artifactId>
97+
<version>0.9.2</version>
98+
<exclusions>
99+
<exclusion>
100+
<artifactId>httpclient</artifactId>
101+
<groupId>org.apache.httpcomponents</groupId>
102+
</exclusion>
103+
<exclusion>
104+
<artifactId>httpcore</artifactId>
105+
<groupId>org.apache.httpcomponents</groupId>
106+
</exclusion>
107+
<exclusion>
108+
<artifactId>slf4j-api</artifactId>
109+
<groupId>org.slf4j</groupId>
110+
</exclusion>
111+
</exclusions>
112+
</dependency>
113+
114+
115+
<dependency>
116+
<groupId>org.apache.hive</groupId>
117+
<artifactId>inceptor-streaming</artifactId>
118+
<version>3.1.3</version>
119+
<exclusions>
120+
<exclusion>
121+
<artifactId>hadoop-hdfs</artifactId>
122+
<groupId>org.apache.hadoop</groupId>
123+
</exclusion>
124+
<exclusion>
125+
<artifactId>hadoop-common</artifactId>
126+
<groupId>org.apache.hadoop</groupId>
127+
</exclusion>
128+
<exclusion>
129+
<artifactId>hadoop-auth</artifactId>
130+
<groupId>org.apache.hadoop</groupId>
131+
</exclusion>
132+
<exclusion>
133+
<artifactId>hadoop-annotations</artifactId>
134+
<groupId>org.apache.hadoop</groupId>
135+
</exclusion>
136+
<exclusion>
137+
<artifactId>hadoop-yarn-api</artifactId>
138+
<groupId>org.apache.hadoop</groupId>
139+
</exclusion>
140+
<exclusion>
141+
<artifactId>hadoop-yarn-common</artifactId>
142+
<groupId>org.apache.hadoop</groupId>
143+
</exclusion>
144+
<exclusion>
145+
<artifactId>hive-serde</artifactId>
146+
<groupId>org.apache.hive</groupId>
147+
</exclusion>
148+
<exclusion>
149+
<artifactId>calcite-core</artifactId>
150+
<groupId>org.apache.calcite</groupId>
151+
</exclusion>
152+
<exclusion>
153+
<artifactId>calcite-avatica</artifactId>
154+
<groupId>org.apache.calcite</groupId>
155+
</exclusion>
156+
<exclusion>
157+
<artifactId>derby</artifactId>
158+
<groupId>org.apache.derby</groupId>
159+
</exclusion>
160+
<exclusion>
161+
<groupId>org.xerial.snappy</groupId>
162+
<artifactId>snappy-java</artifactId>
163+
</exclusion>
164+
</exclusions>
165+
</dependency>
166+
167+
23168
<dependency>
24169
<groupId>com.dtstack.chunjun</groupId>
25170
<artifactId>chunjun-connector-jdbc-base</artifactId>
@@ -29,8 +174,9 @@
29174
<dependency>
30175
<groupId>org.apache.hadoop</groupId>
31176
<artifactId>hadoop-mapreduce-client-core</artifactId>
32-
<version>2.7.5</version>
177+
<version>${hadoop.version}</version>
33178
</dependency>
179+
34180
</dependencies>
35181

36182
<build>
@@ -58,14 +204,82 @@
58204
</filter>
59205
</filters>
60206
<relocations>
207+
208+
<relocation>
209+
<pattern>com.facebook.fb303</pattern>
210+
<shadedPattern>shade.inceptor.com.facebook.fb303</shadedPattern>
211+
</relocation>
212+
213+
<relocation>
214+
<pattern>org.apache.hadoop.hive</pattern>
215+
<shadedPattern>shade.inceptor.org.apache.hadoop.hive</shadedPattern>
216+
<excludes>
217+
<exclude>org.apache.hadoop.hive.ql.io.ProxyLocalFileSystem</exclude>
218+
<exclude>org.apache.hadoop.hive.ql.io.NullScanFileSystem</exclude>
219+
<exclude>org.apache.hadoop.hive.llap.security.*</exclude>
220+
</excludes>
221+
222+
</relocation>
223+
61224
<relocation>
62225
<pattern>org.apache.commons.cli</pattern>
63226
<shadedPattern>shade.inceptor.org.apache.commons.cli</shadedPattern>
64227
</relocation>
228+
229+
65230
<relocation>
66231
<pattern>org.apache.hadoop</pattern>
67232
<shadedPattern>shade.inceptor.org.apache.hadoop</shadedPattern>
233+
<excludes>
234+
<exclude>org.apache.hadoop.security.GroupMappingServiceProvider</exclude>
235+
<exclude>org.apache.hadoop.fs.*</exclude>
236+
237+
<exclude>org.apache.hadoop.conf.*</exclude>
238+
239+
<exclude>org.apache.hadoop.hdfs.*</exclude>
240+
<exclude>org.apache.hadoop.hdfs.protocol.*</exclude>
241+
<exclude>org.apache.hadoop.hdfs.protocol.datatransfer.*</exclude>
242+
<exclude>org.apache.hadoop.hdfs.protocol.proto.*</exclude>
243+
<exclude>org.apache.hadoop.hdfs.protocolPB.*</exclude>
244+
<exclude>org.apache.hadoop.mapred.*</exclude>
245+
<exclude>org.apache.hadoop.mapreduce.*</exclude>
246+
<exclude>org.apache.hadoop.filecache.*</exclude>
247+
248+
<exclude>org.apache.hadoop.io.retry.*</exclude>
249+
<exclude>org.apache.hadoop.hdfs.server.*</exclude>
250+
<exclude>org.apache.hadoop.hdfs.server.namenode.*</exclude>
251+
<exclude>org.apache.hadoop.hdfs.server.namenode.ha.*</exclude>
252+
<exclude>org.apache.hadoop.hdfs.server.protocol.*</exclude>
253+
<exclude>org.apache.hadoop.util.NativeCrc32</exclude>
254+
<exclude>org.apache.hadoop.util.DataChecksum*</exclude>
255+
</excludes>
256+
</relocation>
257+
258+
<relocation>
259+
<pattern>org.apache.hadoop.mapred.FileOutputFormat</pattern>
260+
<shadedPattern>shade.inceptor.org.apache.hadoop.mapred.FileOutputFormat</shadedPattern>
261+
</relocation>
262+
263+
<relocation>
264+
<pattern>org.apache.hadoop.mapred.TextInputFormat</pattern>
265+
<shadedPattern>shade.inceptor.org.apache.hadoop.mapred.TextInputFormat</shadedPattern>
266+
</relocation>
267+
268+
<relocation>
269+
<pattern>org.apache.hadoop.mapred.LineRecordReader</pattern>
270+
<shadedPattern>shade.inceptor.org.apache.hadoop.mapred.LineRecordReader</shadedPattern>
271+
</relocation>
272+
273+
<relocation>
274+
<pattern>org.apache.hadoop.mapred.KeyValueLineRecordReader</pattern>
275+
<shadedPattern>shade.inceptor.org.apache.hadoop.mapred.KeyValueLineRecordReader</shadedPattern>
276+
</relocation>
277+
278+
<relocation>
279+
<pattern>org.apache.hadoop.hdfs.HAUtil</pattern>
280+
<shadedPattern>shade.inceptor.org.apache.hadoop.hdfs.HAUtil</shadedPattern>
68281
</relocation>
282+
69283
<relocation>
70284
<pattern>org.apache.hive</pattern>
71285
<shadedPattern>shade.inceptor.org.apache.hive</shadedPattern>
@@ -82,6 +296,10 @@
82296
<pattern>com.esotericsoftware.kryo</pattern>
83297
<shadedPattern>shade.inceptor.com.esotericsoftware.kryo</shadedPattern>
84298
</relocation>
299+
<relocation>
300+
<pattern>com.google.protobuf</pattern>
301+
<shadedPattern>shade.inceptor.com.google.protobuf</shadedPattern>
302+
</relocation>
85303
</relocations>
86304
<transformers>
87305
<transformer

‎chunjun-connectors/chunjun-connector-inceptor/src/main/java/com/dtstack/chunjun/connector/inceptor/conf/InceptorConf.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@
2323

2424
public class InceptorConf extends JdbcConf {
2525

26+
private String driverType = "HDFS";
27+
2628
private String partition;
2729

2830
private String partitionType;
@@ -52,4 +54,12 @@ public Map<String, Object> getHadoopConfig() {
5254
public void setHadoopConfig(Map<String, Object> hadoopConfig) {
5355
this.hadoopConfig = hadoopConfig;
5456
}
57+
58+
public String getDriverType() {
59+
return driverType;
60+
}
61+
62+
public void setDriverType(String driverType) {
63+
this.driverType = driverType;
64+
}
5565
}

0 commit comments

Comments
 (0)