Skip to content

Commit fe458a9

Browse files
yanghuaiGityanghuaiGit
committed
Merge branch '1.10_release' into git_1.10_release
# Conflicts: # docs/quickstart.md # flinkx-core/src/main/java/com/dtstack/flinkx/util/RangeSplitUtil.java # flinkx-kafka/flinkx-kafka-writer/src/main/java/com/dtstack/flinkx/kafka/writer/KafkaWriter.java # flinkx-kafka10/flinkx-kafka10-writer/src/main/java/com/dtstack/flinkx/kafka10/writer/Kafka10Writer.java # flinkx-kafka11/flinkx-kafka11-writer/src/main/java/com/dtstack/flinkx/kafka11/writer/Kafka11Writer.java # flinkx-launcher/src/main/java/com/dtstack/flinkx/launcher/perJob/PerJobClusterClientBuilder.java # flinkx-launcher/src/main/resources/log4j.properties # flinkx-postgresql/flinkx-postgresql-core/src/main/java/com/dtstack/flinkx/postgresql/PostgresqlTypeConverter.java # flinkx-rdb/flinkx-rdb-core/src/main/java/com/dtstack/flinkx/rdb/util/DbUtil.java # flinkx-rdb/flinkx-rdb-writer/src/main/java/com/dtstack/flinkx/rdb/outputformat/JdbcOutputFormat.java # flinkx-test/src/main/java/com/dtstack/flinkx/test/LocalTest.java # pom.xml
2 parents 7d140cd + b1ad219 commit fe458a9

File tree

262 files changed

+7376
-1978
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

262 files changed

+7376
-1978
lines changed

‎.gitignore‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,5 @@ nohup.out
1010
flinkconf/
1111
hadoopconf/
1212
/default_task_id_output
13-
/syncplugins
13+
/syncplugins
14+
/ci/

‎README.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -66,12 +66,10 @@ The following databases are currently supported:
6666
| | Hive | | [doc](docs/offline/writer/hivewriter.md) |
6767
| Stream Synchronization | Kafka | [doc](docs/realTime/reader/kafkareader.md) | [doc](docs/realTime/writer/kafkawriter.md) |
6868
| | EMQX | [doc](docs/realTime/reader/emqxreader.md) | [doc](docs/realTime/writer/emqxwriter.md) |
69-
| | RestApi | [doc](docs/realTime/reader/restapireader.md) | [doc](docs/realTime/writer/restapiwriter.md) |
69+
| | RestApi || [doc](docs/realTime/writer/restapiwriter.md) |
7070
| | MySQL Binlog | [doc](docs/realTime/reader/binlogreader.md) | |
7171
| | MongoDB Oplog | [doc](docs/realTime/reader/mongodboplogreader.md)| |
7272
| | PostgreSQL WAL | [doc](docs/realTime/reader/pgwalreader.md) | |
73-
| | Oracle Logminer| Coming Soon | |
74-
| | SqlServer CDC | Coming Soon | |
7573

7674
# Quick Start
7775

@@ -89,6 +87,10 @@ Please click [Statistics Metric](docs/statistics.md)
8987

9088
Please click [Kerberos](docs/kerberos.md)
9189

90+
# Questions
91+
92+
Please click [Questions](docs/questions.md)
93+
9294
# How to contribute FlinkX
9395

9496
Please click [Contribution](docs/contribution.md)

‎README_CH.md‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,12 +77,10 @@ FlinkX目前支持下面这些数据库：
7777
| | Hive | | [doc](docs/offline/writer/hivewriter.md) |
7878
| Stream Synchronization | Kafka | [doc](docs/realTime/reader/kafkareader.md) | [doc](docs/realTime/writer/kafkawriter.md) |
7979
| | EMQX | [doc](docs/realTime/reader/emqxreader.md) | [doc](docs/realTime/writer/emqxwriter.md) |
80-
| | RestApi | [doc](docs/realTime/reader/restapireader.md) | [doc](docs/realTime/writer/restapiwriter.md) |
80+
| | RestApi | | [doc](docs/realTime/writer/restapiwriter.md) |
8181
| | MySQL Binlog | [doc](docs/realTime/reader/binlogreader.md) | |
8282
| | MongoDB Oplog | [doc](docs/realTime/reader/mongodboplogreader.md)| |
8383
| | PostgreSQL WAL | [doc](docs/realTime/reader/pgwalreader.md) | |
84-
| | Oracle Logminer| Coming Soon | |
85-
| | SqlServer CDC | Coming Soon | |
8684

8785
# 快速开始
8886

@@ -100,6 +98,10 @@ FlinkX目前支持下面这些数据库：
10098

10199
请点击[Kerberos](docs/kerberos.md)
102100

101+
# Questions
102+
103+
请点击[Questions](docs/questions.md)
104+
103105
# 如何贡献FlinkX
104106

105107
请点击[如何贡献FlinkX](docs/contribution.md)

‎bin/install_jars.bat‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,3 +6,6 @@ call mvn install:install-file -DgroupId=com.esen.jdbc -DartifactId=gbase -Dversi
66

77
call mvn install:install-file -DgroupId=dm.jdbc.driver -DartifactId=dm7 -Dversion=18.0.0 -Dpackaging=jar -Dfile=../jars/Dm7JdbcDriver18.jar
88

9+
call mvn install:install-file -DgroupId=com.kingbase8 -DartifactId=kingbase8 -Dversion=8.2.0 -Dpackaging=jar -Dfile=../jars/kingbase8-8.2.0.jar
10+
11+
call mvn install:install-file -DgroupId=fakepath -DartifactId=vertica-jdbc -Dversion=9.1.1-0 -Dpackaging=jar -Dfile=../jars/vertica-jdbc-9.1.1-0.jar

‎bin/install_jars.sh‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,4 +10,10 @@ mvn install:install-file -DgroupId=com.github.noraui -DartifactId=ojdbc8 -Dversi
1010
mvn install:install-file -DgroupId=com.esen.jdbc -DartifactId=gbase -Dversion=8.3.81.53 -Dpackaging=jar -Dfile=../jars/gbase-8.3.81.53.jar
1111

1212
## dm driver
13-
mvn install:install-file -DgroupId=dm.jdbc.driver -DartifactId=dm7 -Dversion=18.0.0 -Dpackaging=jar -Dfile=../jars/Dm7JdbcDriver18.jar
13+
mvn install:install-file -DgroupId=dm.jdbc.driver -DartifactId=dm7 -Dversion=18.0.0 -Dpackaging=jar -Dfile=../jars/Dm7JdbcDriver18.jar
14+
15+
## kingbase driver
16+
mvn install:install-file -DgroupId=com.kingbase8 -DartifactId=kingbase8 -Dversion=8.2.0 -Dpackaging=jar -Dfile=../jars/kingbase8-8.2.0.jar
17+
18+
## vertica driver
19+
mvn install:install-file -DgroupId=fakepath -DartifactId=vertica-jdbc -Dversion=9.1.1-0 -Dpackaging=jar -Dfile=../jars/vertica-jdbc-9.1.1-0.jar

‎docs/example/kafka09_stream.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"job": {
3+
"content": [
4+
{
5+
"reader": {
6+
"parameter": {
7+
"topic": "kafka09",
8+
"groupId": "default",
9+
"codec": "text",
10+
"encoding": "UTF-8",
11+
"blankIgnore": false,
12+
"consumerSettings": {
13+
"zookeeper.connect": "localhost:2181/kafka09"
14+
}
15+
},
16+
"name": "kafka09reader"
17+
},
18+
"writer": {
19+
"parameter": {
20+
"print": true
21+
},
22+
"name": "streamwriter"
23+
}
24+
}
25+
],
26+
"setting": {
27+
"restore": {
28+
"isRestore": false,
29+
"isStream": true
30+
},
31+
"speed": {
32+
"channel": 1
33+
}
34+
}
35+
}
36+
}

‎docs/example/kafka10_stream.json‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
{
2+
"job": {
3+
"content": [
4+
{
5+
"reader": {
6+
"parameter": {
7+
"topic": "kafka10",
8+
"groupId": "default",
9+
"codec": "text",
10+
"encoding": "UTF-8",
11+
"blankIgnore": false,
12+
"consumerSettings": {
13+
"bootstrap.servers": "localhost:9092"
14+
}
15+
},
16+
"name": "kafka10reader"
17+
},
18+
"writer": {
19+
"parameter": {
20+
"print": true
21+
},
22+
"name": "streamwriter"
23+
}
24+
}
25+
],
26+
"writer": {
27+
"parameter": {
28+
"print": true
29+
},
30+
"name": "streamwriter"
31+
}
32+
},
33+
"setting": {
34+
"restore": {
35+
"isRestore": false,
36+
"isStream": true
37+
},
38+
"speed": {
39+
"channel": 1
40+
}
41+
}
42+
}

‎docs/example/kafka11_stream.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"job": {
3+
"content": [
4+
{
5+
"reader": {
6+
"parameter": {
7+
"topic": "kafka11",
8+
"groupId": "default",
9+
"codec": "text",
10+
"encoding": "UTF-8",
11+
"blankIgnore": false,
12+
"consumerSettings": {
13+
"bootstrap.servers": "localhost:9092"
14+
}
15+
},
16+
"name": "kafka11reader"
17+
},
18+
"writer": {
19+
"parameter": {
20+
"print": true
21+
},
22+
"name": "streamwriter"
23+
}
24+
}
25+
],
26+
"setting": {
27+
"restore": {
28+
"isRestore": false,
29+
"isStream": true
30+
},
31+
"speed": {
32+
"channel": 1
33+
}
34+
}
35+
}
36+
}

‎docs/example/kafka_hive.json‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,52 @@
1+
{
2+
"job": {
3+
"content": [
4+
{
5+
"reader" : {
6+
"parameter" : {
7+
"topic" : "test",
8+
"mode": "timestamp",
9+
"timestamp": 1609812275000,
10+
"codec": "text",
11+
"consumerSettings" : {
12+
"bootstrap.servers" : "ip1:9092,ip2:9092,ip3:9092"
13+
}
14+
},
15+
"name" : "kafkareader"
16+
},
17+
"writer": {
18+
"parameter" : {
19+
"jdbcUrl" : "jdbc:hive2://ip:10000/test",
20+
"fileType" : "parquet",
21+
"writeMode" : "overwrite",
22+
"compress" : "",
23+
"charsetName" : "UTF-8",
24+
"maxFileSize" : 1073741824,
25+
"tablesColumn" : "{\"message\":[{\"part\":false,\"comment\":\"\",\"type\":\"string\",\"key\":\"message\"}]}",
26+
"partition" : "pt",
27+
"partitionType" : "DAY",
28+
"defaultFS" : "hdfs://ns",
29+
"hadoopConfig": {
30+
"dfs.ha.namenodes.ns": "nn1,nn2",
31+
"dfs.namenode.rpc-address.ns.nn2": "ip1:9000",
32+
"dfs.client.failover.proxy.provider.ns": "org.apache.hadoop.hdfs.server.namenode.ha.ConfiguredFailoverProxyProvider",
33+
"dfs.namenode.rpc-address.ns.nn1": "ip2:9000",
34+
"dfs.nameservices": "ns"
35+
}
36+
},
37+
"name" : "hivewriter"
38+
}
39+
}
40+
],
41+
"setting": {
42+
"restore": {
43+
"isRestore": true,
44+
"isStream": true
45+
},
46+
"speed": {
47+
"readerChannel": 3,
48+
"writerChannel": 1
49+
}
50+
}
51+
}
52+
}

‎docs/example/kafka_mysql.json‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,65 @@
1+
{
2+
"job": {
3+
"content": [
4+
{
5+
"reader": {
6+
"parameter": {
7+
"topic": "tudou",
8+
"mode": "timestamp",
9+
"timestamp": 1609812275000,
10+
"offset": "partition:0,offset:0;partition:1,offset:1;partition:2,offset:2",
11+
"codec": "text",
12+
"blankIgnore": false,
13+
"consumerSettings": {
14+
"bootstrap.servers": "ip1:9092,ip2:9092,ip3:9092"
15+
},
16+
"column": ["id","user_id","name"]
17+
},
18+
"name": "kafkareader"
19+
},
20+
"writer": {
21+
"name": "mysqlwriter",
22+
"parameter": {
23+
"username": "root",
24+
"password": "abc123",
25+
"connection": [
26+
{
27+
"jdbcUrl": "jdbc:mysql://localhost:3306/test",
28+
"table": [
29+
"test"
30+
]
31+
}
32+
],
33+
"preSql": ["truncate table test;"],
34+
"postSql": [],
35+
"writeMode": "insert",
36+
"column": [
37+
{
38+
"name": "id",
39+
"type": "BIGINT"
40+
},
41+
{
42+
"name": "user_id",
43+
"type": "BIGINT"
44+
},
45+
{
46+
"name": "name",
47+
"type": "varchar"
48+
}
49+
]
50+
}
51+
}
52+
}
53+
],
54+
"setting": {
55+
"restore": {
56+
"isRestore": true,
57+
"isStream": true
58+
},
59+
"speed": {
60+
"readerChannel": 3,
61+
"writerChannel": 1
62+
}
63+
}
64+
}
65+
}

0 commit comments

Comments
 (0)