Skip to content

Commit 4566ece

Browse files
todd5167todd5167
committed
oracle mysql side table doc
1 parent 667e2d4 commit 4566ece

File tree

3 files changed

+203
-80
lines changed

3 files changed

+203
-80
lines changed

‎docs/plugin/mysqlSide.md‎

Lines changed: 104 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,15 @@
11

22
## 1.格式：
3+
4+
通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段，
5+
维表JOIN的条件必须与`keyInfo`字段一致。
6+
37
```
48
CREATE TABLE tableName(
59
colName cloType,
610
...
711
PRIMARY KEY(keyInfo),
8-
PERIOD FOR SYSTEM_TIME
12+
PERIOD FOR SYSTEM_TIME
913
)WITH(
1014
type='mysql',
1115
url='jdbcUrl',
@@ -24,62 +28,117 @@
2428
mysql-5.6.35
2529

2630
## 3.表结构定义
27-
28-
|参数名称|含义|
29-
|----|---|
30-
| tableName | mysql表名称|
31-
| colName | 列名称|
32-
| colType | 列类型 [colType支持的类型](docs/colType.md)|
33-
| PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息|
34-
| PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开|
31+
32+
[关系型数据库维表参数](docs/rdbSideParams.md)
3533

36-
## 4.参数
34+
## 4.样例
3735

38-
|参数名称|含义|是否必填|默认值|
39-
|----|---|---|----|
40-
| type | 表明维表的类型 mysql |是||
41-
| url | 连接mysql数据库 jdbcUrl |是||
42-
| userName | mysql连接用户名 |是||
43-
| password | mysql连接密码|是||
44-
| tableName | mysql表名称|是||
45-
| tableName | mysql 的表名称|是||
46-
| cache | 维表缓存策略(NONE/LRU)|否|NONE|
47-
| partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false|
48-
49-
----------
50-
> 缓存策略
51-
* NONE: 不做内存缓存
52-
* LRU:
53-
* cacheSize: 缓存的条目数量
54-
* cacheTTLMs:缓存的过期时间(ms)
55-
* cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。
56-
* asyncCapacity:异步请求容量，默认1000
57-
* asyncTimeout：异步请求超时时间，默认10000毫秒
36+
### ALL全量维表定义
37+
```
38+
// 定义全量维表
39+
CREATE TABLE sideTable(
40+
id INT,
41+
name VARCHAR,
42+
PRIMARY KEY(id) ,
43+
PERIOD FOR SYSTEM_TIME
44+
)WITH(
45+
type ='mysql',
46+
url ='jdbc:mysql://172.16.10.204:3306/mqtest',
47+
userName ='dtstack',
48+
password ='1abc123',
49+
tableName ='test_mysql_10',
50+
cache ='ALL',
51+
cacheTTLMs ='60000',
52+
parallelism ='2'
53+
);
5854
59-
## 5.样例
6055
```
61-
create table sideTable(
62-
channel varchar,
63-
xccount int,
64-
PRIMARY KEY(channel),
56+
### LRU异步维表定义
57+
58+
```
59+
CREATE TABLE sideTable(
60+
id INT,
61+
name VARCHAR,
62+
PRIMARY KEY(id) ,
6563
PERIOD FOR SYSTEM_TIME
6664
)WITH(
67-
type='mysql',
68-
url='jdbc:mysql://172.16.8.104:3306/test?charset=utf8',
69-
userName='dtstack',
70-
password='abc123',
71-
tableName='sidetest',
65+
type ='mysql',
66+
url ='jdbc:mysql://172.16.10.204:3306/mqtest',
67+
userName ='dtstack',
68+
password ='1abc123',
69+
tableName ='yctest_mysql_10',
70+
partitionedJoin ='false',
7271
cache ='LRU',
7372
cacheSize ='10000',
7473
cacheTTLMs ='60000',
75-
cacheMode='unordered',
76-
asyncCapacity='1000',
77-
asyncTimeout='10000'
78-
parallelism ='1',
79-
partitionedJoin='false'
74+
asyncPoolSize ='3',
75+
parallelism ='2'
8076
);
8177
78+
```
79+
8280

81+
### MySQL异步维表关联
8382
```
83+
CREATE TABLE MyTable(
84+
id int,
85+
name varchar
86+
)WITH(
87+
type ='kafka11',
88+
bootstrapServers ='172.16.8.107:9092',
89+
zookeeperQuorum ='172.16.8.107:2181/kafka',
90+
offsetReset ='latest',
91+
topic ='cannan_yctest01',
92+
timezone='Asia/Shanghai',
93+
enableKeyPartitions ='false',
94+
topicIsPattern ='false',
95+
parallelism ='1'
96+
);
97+
98+
CREATE TABLE MyResult(
99+
id INT,
100+
name VARCHAR
101+
)WITH(
102+
type ='mysql',
103+
url ='jdbc:mysql://172.16.10.204:3306/mqtest',
104+
userName ='dtstack',
105+
password ='1abc123',
106+
tableName ='yctest_mysql_mq',
107+
updateMode ='append',
108+
parallelism ='1',
109+
batchSize ='100',
110+
batchWaitInterval ='1000'
111+
);
112+
113+
CREATE TABLE sideTable(
114+
id INT,
115+
name VARCHAR,
116+
PRIMARY KEY(id) ,
117+
PERIOD FOR SYSTEM_TIME
118+
)WITH(
119+
type ='mysql',
120+
url ='jdbc:mysql://172.16.10.204:3306/mqtest',
121+
userName ='dtstack',
122+
password ='1abc123',
123+
tableName ='yctest_mysql_10',
124+
partitionedJoin ='false',
125+
cache ='LRU',
126+
cacheSize ='10000',
127+
cacheTTLMs ='60000',
128+
asyncPoolSize ='3',
129+
parallelism ='1'
130+
);
84131
132+
insert
133+
into
134+
MyResult
135+
select
136+
m.id,
137+
s.name
138+
from
139+
MyTable m
140+
join
141+
sideTable s
142+
on m.id=s.id;
85143
144+
```

‎docs/plugin/oracleSide.md‎

Lines changed: 98 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,13 @@
11

22
## 1.格式：
3+
4+
通过建表语句中的` PERIOD FOR SYSTEM_TIME`将表标识为维表，其中`PRIMARY KEY(keyInfo)`中的keyInfo，表示用来和源表进行关联的字段，
5+
维表JOIN的条件必须与`keyInfo`字段一致。
6+
7+
此外, 针对oracle char类型自动补齐的特性，我们允许为char指定长度，确保维表查询时能够匹配数据。
8+
9+
10+
注意：Oracle维表使用的字段大小写，需要和Oracle中定义的保持一致。
311
```
412
CREATE TABLE tableName(
513
colName cloType,
@@ -22,43 +30,35 @@
2230
# 2.支持版本
2331
10g 11g
2432

25-
## 3.表结构定义
26-
27-
|参数名称|含义|
28-
|----|---|
29-
| tableName | oracle表名称|
30-
| colName | 列名称|
31-
| colType | 列类型 [colType支持的类型](docs/colType.md)|
32-
| PERIOD FOR SYSTEM_TIME | 关键字表明该定义的表为维表信息|
33-
| PRIMARY KEY(keyInfo) | 维表主键定义;多个列之间用逗号隔开|
34-
35-
## 4.参数
36-
37-
|参数名称|含义|是否必填|默认值|
38-
|----|---|---|----|
39-
| type | 表明维表的类型 oracle |是||
40-
| url | 连接oracle数据库 jdbcUrl |是||
41-
| userName | oracle连接用户名 |是||
42-
| password | oracle连接密码|是||
43-
| tableName | oracle表名称|是||
44-
| schema | oracle 的schema|否|当前登录用户|
45-
| cache | 维表缓存策略(NONE/LRU)|否|NONE|
46-
| partitionedJoin | 是否在維表join之前先根据 設定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false|
47-
48-
----------
49-
> 缓存策略
50-
* NONE: 不做内存缓存
51-
* LRU:
52-
* cacheSize: 缓存的条目数量
53-
* cacheTTLMs:缓存的过期时间(ms)
54-
* cacheMode: (unordered|ordered)异步加载是有序还是无序,默认有序。
55-
* asyncCapacity:异步请求容量，默认1000
56-
* asyncTimeout：异步请求超时时间，默认10000毫秒
57-
58-
## 5.样例
33+
## 3. 表结构定义
34+
[关系型数据库维表参数](docs/rdbSideParams.md)
35+
36+
## 4.样例
37+
38+
### ALL全量维表定义
39+
```
40+
CREATE TABLE sideTable(
41+
ID char(20), // oracle定义了char(20)
42+
NAME varchar,
43+
PRIMARY KEY (ID),
44+
PERIOD FOR SYSTEM_TIME
45+
)WITH(
46+
type='oracle',
47+
url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe',
48+
userName = 'system',
49+
password = 'oracle',
50+
tableName = 'SIDETEST1',
51+
schema = 'dtstack',
52+
cache = 'ALL',
53+
cacheTTLMs ='60000'
54+
);
55+
```
56+
57+
### LRU异步维表定义
58+
5959
```
6060
create table sideTable(
61-
channel varchar,
61+
channel char,
6262
xccount int,
6363
PRIMARY KEY(channel),
6464
PERIOD FOR SYSTEM_TIME
@@ -79,7 +79,70 @@ create table sideTable(
7979
schema = 'MQTEST'
8080
);
8181
82+
```
83+
84+
### Oracle异步维表关联
85+
86+
```aidl
87+
CREATE TABLE MyTable(
88+
id varchar,
89+
name varchar
90+
--ts timestamp,
91+
--tsDate Date
92+
)WITH(
93+
type ='kafka11',
94+
bootstrapServers ='172.16.8.107:9092',
95+
zookeeperQuorum ='172.16.8.107:2181/kafka',
96+
offsetReset ='latest',
97+
topic ='mqTest01',
98+
timezone='Asia/Shanghai',
99+
topicIsPattern ='false',
100+
parallelism ='1'
101+
);
102+
103+
104+
CREATE TABLE sideTable(
105+
ID char(20),
106+
NAME varchar,
107+
PRIMARY KEY (ID),
108+
PERIOD FOR SYSTEM_TIME
109+
)WITH(
110+
type='oracle',
111+
url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe',
112+
userName = 'system',
113+
password = 'oracle',
114+
tableName = 'SIDETEST1',
115+
--schema = 'dtstack',
116+
cache = 'LRU',
117+
asyncPoolSize ='3'
118+
);
119+
120+
121+
CREATE TABLE MyResult(
122+
NAME varchar,
123+
ID char(20),
124+
PRIMARY KEY (ID)
125+
)WITH(
126+
--type ='console',
127+
type='oracle',
128+
url = 'jdbc:oracle:thin:@172.16.8.178:1521:xe',
129+
userName = 'system',
130+
password = 'oracle',
131+
tableName = 'SINK_TEST',
132+
batchSize='1'
133+
);
134+
135+
82136
137+
INSERT INTO MyResult
138+
SELECT
139+
s.ID as ID,
140+
m.name as NAME
141+
FROM MyTable m
142+
LEFT JOIN
143+
sideTable s
144+
ON
145+
m.id=s.ID
83146
```
84147

85148

‎docs/plugin/rdbSideParams.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
| schema| 表空间|否||
1515
| cache | 维表缓存策略(NONE/LRU/ALL)|否|LRU|
1616
| partitionedJoin | 是否在維表join之前先根据设定的key 做一次keyby操作(可以減少维表的数据缓存量)|否|false|
17+
| parallelism | 处理后的数据流并行度|否||
1718

1819
### 缓存策略
1920

0 commit comments

Comments
 (0)