Skip to content

Commit a82f900

Browse files
simenliuxingsimenliuxing
committed
pgsql
2 parents 7ad245c + de9e241 commit a82f900

File tree

18 files changed

+1039
-2
lines changed

18 files changed

+1039
-2
lines changed

‎README.md‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@
1010

1111
# 已支持
1212
* 源表：kafka 0.9，1.x版本
13-
* 维表：mysql, SQlServer,oracle, hbase, mongo, redis, cassandra, kudu
14-
* 结果表：mysql, SQlServer, oracle, hbase, elasticsearch5.x, mongo, redis, cassandra, kudu
13+
* 维表：mysql，SQlServer,oracle,hbase，mongo，redis,cassandra, kudu, postgresql
14+
* 结果表：mysql，SQlServer,oracle,hbase，elasticsearch5.x，mongo，redis,cassandra, kudu, postgresql
1515

1616
# 后续开发计划
1717
* 增加SQL支持CEP
@@ -155,6 +155,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack
155155
* [redis 结果表插件](docs/redisSink.md)
156156
* [cassandra 结果表插件](docs/cassandraSink.md)
157157
* [kudu 结果表插件](docs/kuduSink.md)
158+
* [postgresql 结果表插件](docs/postgresqlSink.md)
158159

159160
### 2.3 维表插件
160161
* [hbase 维表插件](docs/hbaseSide.md)
@@ -163,6 +164,7 @@ sh submit.sh -sql D:\sideSql.txt -name xctest -remoteSqlPluginPath /opt/dtstack
163164
* [redis 维表插件](docs/redisSide.md)
164165
* [cassandra 维表插件](docs/cassandraSide.md)
165166
* [kudu 维表插件](docs/kuduSide.md)
167+
* [postgresql 维表插件](docs/postgresqlSide.md)
166168

167169
## 3 性能指标(新增)
168170

‎docs/postgresqlSide.md‎

Lines changed: 80 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
2+
## 1.格式：
3+
```
4+
CREATE TABLE tableName(
5+
colName cloType,
6+
...
7+
PRIMARY KEY(keyInfo),
8+
PERIOD FOR SYSTEM_TIME
9+
)WITH(
10+
type='postgresql',
11+
url='jdbcUrl',
12+
userName='dbUserName',
13+
password='dbPwd',
14+
tableName='tableName',
15+
cache ='LRU',
16+
cacheSize ='10000',
17+
cacheTTLMs ='60000',
18+
parallelism ='1',
19+
partitionedJoin='false'
20+
);
21+
```
22+
23+
# 2.支持版本
24+
postgresql-8.2+
25+
26+
## 3.表结构定义
27+
28+
|参数名称|含义|
29+
|----|---|
30+
| tableName | 注册到flink的表名称(可选填;不填默认和hbase对应的表名称相同)|
31+
| colName | 列名称|
32+
| colType | 列类型 [colType支持的类型](colType.md)|
33+
| PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息|
34+
| PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开|
35+
36+
## 4.参数
37+
38+
|参数名称|含义|是否必填|默认值|
39+
|----|---|---|----|
40+
| type | 表明维表的类型[postgresql] |是||
41+
| url | 连接postgresql数据库 jdbcUrl |是||
42+
| userName | postgresql连接用户名 |是||
43+
| password | postgresql连接密码|是||
44+
| tableName | postgresql表名称|是||
45+
| tableName | postgresql 的表名称|是||
46+
| cache | 维表缓存策略(NONE/LRU/ALL)|否|NONE|
47+
| partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false|
48+
49+
----------
50+
> 缓存策略
51+
* NONE: 不做内存缓存
52+
* LRU:
53+
* cacheSize: 缓存的条目数量
54+
* cacheTTLMs:缓存的过期时间(ms)
55+
56+
57+
## 5.样例
58+
```
59+
create table sideTable(
60+
channel varchar,
61+
xccount int,
62+
PRIMARY KEY(channel),
63+
PERIOD FOR SYSTEM_TIME
64+
)WITH(
65+
type='postgresql',
66+
url='jdbc:postgresql://localhost:9001/test?sslmode=disable',
67+
userName='dtstack',
68+
password='abc123',
69+
tableName='sidetest',
70+
cache ='LRU',
71+
cacheSize ='10000',
72+
cacheTTLMs ='60000',
73+
parallelism ='1',
74+
partitionedJoin='false'
75+
);
76+
77+
78+
```
79+
80+

‎docs/postgresqlSink.md‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
## 1.格式：
2+
```
3+
CREATE TABLE tableName(
4+
colName colType,
5+
...
6+
colNameX colType
7+
)WITH(
8+
type ='postgresql',
9+
url ='jdbcUrl',
10+
userName ='userName',
11+
password ='pwd',
12+
tableName ='tableName',
13+
parallelism ='parllNum'
14+
);
15+
16+
```
17+
18+
## 2.支持版本
19+
postgresql-8.2+
20+
21+
## 3.表结构定义
22+
23+
|参数名称|含义|
24+
|----|---|
25+
| tableName| 在 sql 中使用的名称;即注册到flink-table-env上的名称|
26+
| colName | 列名称|
27+
| colType | 列类型 [colType支持的类型](colType.md)|
28+
29+
## 4.参数：
30+
31+
|参数名称|含义|是否必填|默认值|
32+
|----|----|----|----|
33+
| type |表明 输出表类型[postgresql]|是||
34+
| url | 连接postgresql数据库 jdbcUrl |是||
35+
| userName | postgresql连接用户名 |是||
36+
| password | postgresql连接密码|是||
37+
| tableName | postgresqll表名称|是||
38+
| parallelism | 并行度设置|否|1|
39+
| isUpsert | 使用upsert模式插入数据(版本9.5之后才支持upsert) |否|false
40+
| keyField | 设置更新主键字段名（isupsert为true时为必填项）|否|
41+
42+
## 5.样例：
43+
```
44+
CREATE TABLE MyResult(
45+
channel VARCHAR,
46+
pv VARCHAR
47+
)WITH(
48+
type ='postgresql',
49+
url ='jdbc:postgresql://localhost:9001/test?sslmode=disable',
50+
userName ='dtstack',
51+
password ='abc123',
52+
tableName ='pv2',
53+
parallelism ='1'
54+
)
55+
```

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
<module>oracle</module>
2525
<module>cassandra</module>
2626
<module>kudu</module>
27+
<module>postgresql</module>
2728
</modules>
2829

2930

‎postgresql/pom.xml‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>flink.sql</artifactId>
7+
<groupId>com.dtstack.flink</groupId>
8+
<version>1.0-SNAPSHOT</version>
9+
<relativePath>../pom.xml</relativePath>
10+
</parent>
11+
<modelVersion>4.0.0</modelVersion>
12+
<packaging>pom</packaging>
13+
<artifactId>sql.postgresql</artifactId>
14+
15+
<modules>
16+
<module>postgresql-side</module>
17+
<module>postgresql-sink</module>
18+
</modules>
19+
<properties>
20+
<sql.core.version>1.0-SNAPSHOT</sql.core.version>
21+
<postgresql.version>42.2.2</postgresql.version>
22+
</properties>
23+
24+
<dependencies>
25+
<dependency>
26+
<groupId>junit</groupId>
27+
<artifactId>junit</artifactId>
28+
<version>3.8.1</version>
29+
<scope>test</scope>
30+
</dependency>
31+
32+
<dependency>
33+
<groupId>com.dtstack.flink</groupId>
34+
<artifactId>sql.core</artifactId>
35+
<version>${sql.core.version}</version>
36+
<scope>provided</scope>
37+
</dependency>
38+
<dependency>
39+
<groupId>org.postgresql</groupId>
40+
<artifactId>postgresql</artifactId>
41+
<version>${postgresql.version}</version>
42+
</dependency>
43+
</dependencies>
44+
</project>

‎postgresql/postgresql-side/pom.xml‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>sql.postgresql</artifactId>
7+
<groupId>com.dtstack.flink</groupId>
8+
<version>1.0-SNAPSHOT</version>
9+
<relativePath>../pom.xml</relativePath>
10+
</parent>
11+
<modelVersion>4.0.0</modelVersion>
12+
13+
<artifactId>sql.side.postgresql</artifactId>
14+
<version>1.0-SNAPSHOT</version>
15+
<name>postgresql-side</name>
16+
<packaging>pom</packaging>
17+
18+
<modules>
19+
<module>postgresql-side-core</module>
20+
<module>postgresql-async-side</module>
21+
<module>postgresql-all-side</module>
22+
</modules>
23+
24+
25+
<properties>
26+
<rdb.side.version>1.0-SNAPSHOT</rdb.side.version>
27+
</properties>
28+
29+
<dependencies>
30+
<dependency>
31+
<groupId>com.dtstack.flink</groupId>
32+
<artifactId>sql.side.rdb</artifactId>
33+
<version>${rdb.side.version}</version>
34+
</dependency>
35+
</dependencies>
36+
37+
38+
</project>

‎postgresql/postgresql-side/postgresql-all-side/pom.xml‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>sql.side.postgresql</artifactId>
7+
<groupId>com.dtstack.flink</groupId>
8+
<version>1.0-SNAPSHOT</version>
9+
<relativePath>../pom.xml</relativePath>
10+
</parent>
11+
<modelVersion>4.0.0</modelVersion>
12+
13+
<artifactId>sql.side.all.postgresql</artifactId>
14+
<name>postgresql-all-side</name>
15+
16+
<packaging>jar</packaging>
17+
18+
<properties>
19+
<sql.side.postgresql.core.version>1.0-SNAPSHOT</sql.side.postgresql.core.version>
20+
</properties>
21+
22+
<dependencies>
23+
<dependency>
24+
<groupId>com.dtstack.flink</groupId>
25+
<artifactId>sql.side.postgresql.core</artifactId>
26+
<version>${sql.side.postgresql.core.version}</version>
27+
</dependency>
28+
</dependencies>
29+
30+
<build>
31+
<plugins>
32+
<plugin>
33+
<groupId>org.apache.maven.plugins</groupId>
34+
<artifactId>maven-shade-plugin</artifactId>
35+
<version>1.4</version>
36+
<executions>
37+
<execution>
38+
<phase>package</phase>
39+
<goals>
40+
<goal>shade</goal>
41+
</goals>
42+
<configuration>
43+
<artifactSet>
44+
<excludes>
45+
46+
</excludes>
47+
</artifactSet>
48+
<filters>
49+
<filter>
50+
<artifact>*:*</artifact>
51+
<excludes>
52+
<exclude>META-INF/*.SF</exclude>
53+
<exclude>META-INF/*.DSA</exclude>
54+
<exclude>META-INF/*.RSA</exclude>
55+
</excludes>
56+
</filter>
57+
</filters>
58+
</configuration>
59+
</execution>
60+
</executions>
61+
</plugin>
62+
63+
<plugin>
64+
<artifactId>maven-antrun-plugin</artifactId>
65+
<version>1.2</version>
66+
<executions>
67+
<execution>
68+
<id>copy-resources</id>
69+
<!-- here the phase you need -->
70+
<phase>package</phase>
71+
<goals>
72+
<goal>run</goal>
73+
</goals>
74+
<configuration>
75+
<tasks>
76+
<copy todir="${basedir}/../../../plugins/postgresqlallside">
77+
<fileset dir="target/">
78+
<include name="${project.artifactId}-${project.version}.jar"/>
79+
</fileset>
80+
</copy>
81+
82+
<move file="${basedir}/../../../plugins/postgresqlallside/${project.artifactId}-${project.version}.jar"
83+
tofile="${basedir}/../../../plugins/postgresqlallside/${project.name}-${git.branch}.jar"/>
84+
</tasks>
85+
</configuration>
86+
</execution>
87+
</executions>
88+
</plugin>
89+
</plugins>
90+
</build>
91+
92+
</project>

0 commit comments

Comments
 (0)