Skip to content

Commit de5b214

Browse files
authored
[feat-#1487][iceberg]add iceberg writer (#1489)
1 parent afc7d51 commit de5b214

File tree

16 files changed

+334
-993
lines changed

16 files changed

+334
-993
lines changed

chunjun-connectors/chunjun-connector-iceberg/pom.xml

Lines changed: 40 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -31,40 +31,53 @@
3131
<artifactId>chunjun-connector-iceberg</artifactId>
3232
<name>ChunJun : Connectors : Iceberg</name>
3333

34-
<dependencies>
34+
<properties>
35+
<hadoop.version>2.7.5</hadoop.version>
36+
<iceberg.version>0.13.1</iceberg.version>
37+
</properties>
38+
3539

36-
<!-- flink iceberg connector start -->
37-
<!-- <dependency>-->
38-
<!-- <groupId>org.apache.iceberg</groupId>-->
39-
<!-- <artifactId>iceberg-flink-1.12</artifactId>-->
40-
<!-- <version>0.13.2</version>-->
41-
<!-- <exclusions>-->
42-
<!-- <exclusion>-->
43-
<!-- <artifactId>avro</artifactId>-->
44-
<!-- <groupId>org.apache.avro</groupId>-->
45-
<!-- </exclusion>-->
46-
<!-- </exclusions>-->
47-
<!-- </dependency>-->
40+
<dependencies>
4841
<dependency>
4942
<groupId>org.apache.iceberg</groupId>
50-
<artifactId>iceberg-flink-runtime-1.12</artifactId>
51-
<version>0.13.2</version>
43+
<artifactId>iceberg-flink-runtime-1.13</artifactId>
44+
<version>${iceberg.version}</version>
45+
<exclusions>
46+
<exclusion>
47+
<groupId>org.apache.hadoop</groupId>
48+
<artifactId>hadoop-hdfs</artifactId>
49+
</exclusion>
50+
51+
<exclusion>
52+
<groupId>org.apache.hadoop</groupId>
53+
<artifactId>hadoop-common</artifactId>
54+
</exclusion>
55+
56+
<exclusion>
57+
<groupId>org.apache.hadoop</groupId>
58+
<artifactId>hadoop-client</artifactId>
59+
</exclusion>
60+
</exclusions>
61+
</dependency>
62+
63+
<dependency>
64+
<groupId>org.apache.hadoop</groupId>
65+
<artifactId>hadoop-hdfs</artifactId>
66+
<version>${hadoop.version}</version>
5267
</dependency>
5368

54-
<!-- <dependency>-->
55-
<!-- <groupId>org.apache.hadoop</groupId>-->
56-
<!-- <artifactId>hadoop-client</artifactId>-->
57-
<!-- <version>2.7.3</version>-->
58-
<!-- <exclusions>-->
59-
<!-- <exclusion>-->
60-
<!-- <artifactId>avro</artifactId>-->
61-
<!-- <groupId>org.apache.avro</groupId>-->
62-
<!-- </exclusion>-->
63-
<!-- </exclusions>-->
64-
<!-- </dependency>-->
69+
<dependency>
70+
<groupId>org.apache.hadoop</groupId>
71+
<artifactId>hadoop-common</artifactId>
72+
<version>${hadoop.version}</version>
73+
</dependency>
6574

75+
<dependency>
76+
<groupId>org.apache.hadoop</groupId>
77+
<artifactId>hadoop-client</artifactId>
78+
<version>${hadoop.version}</version>
79+
</dependency>
6680

67-
<!-- flink iceberg connector end -->
6881
</dependencies>
6982

7083
<build>
Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
package com.dtstack.chunjun.connector.iceberg.conf;
2+
3+
import com.dtstack.chunjun.conf.ChunJunCommonConf;
4+
import com.dtstack.chunjun.sink.WriteMode;
5+
6+
import java.util.HashMap;
7+
import java.util.Map;
8+
9+
public class IcebergConf extends ChunJunCommonConf {
10+
private String warehouse;
11+
12+
private String uri;
13+
14+
private Map<String, Object> hadoopConfig = new HashMap<>();
15+
16+
private String database;
17+
18+
private String table;
19+
20+
private String writeMode = WriteMode.APPEND.name();
21+
22+
public void setDatabase(String database) {
23+
this.database = database;
24+
}
25+
26+
public String getDatabase() {
27+
return database;
28+
}
29+
30+
public void setTable(String table) {
31+
this.table = table;
32+
}
33+
34+
public String getTable() {
35+
return table;
36+
}
37+
38+
public void setWarehouse(String warehouse) {
39+
this.warehouse = warehouse;
40+
}
41+
42+
public String getWarehouse() {
43+
return warehouse;
44+
}
45+
46+
public void setUri(String uri) {
47+
this.uri = uri;
48+
}
49+
50+
public String getUri() {
51+
return uri;
52+
}
53+
54+
public void setHadoopConfig(Map<String, Object> hadoopConfig) {
55+
this.hadoopConfig = hadoopConfig;
56+
}
57+
58+
public Map<String, Object> getHadoopConfig() {
59+
return hadoopConfig;
60+
}
61+
62+
public String getWriteMode() {
63+
return writeMode;
64+
}
65+
66+
public void setWriteMode(String writeMode) {
67+
this.writeMode = writeMode;
68+
}
69+
}

chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergReaderConf.java

Lines changed: 0 additions & 69 deletions
This file was deleted.

chunjun-connectors/chunjun-connector-iceberg/src/main/java/com/dtstack/chunjun/connector/iceberg/conf/IcebergWriterConf.java

Lines changed: 0 additions & 96 deletions
This file was deleted.

0 commit comments

Comments
 (0)