Skip to content

Commit 9168f4c

Browse files
TrafalgarLuoTrafalgarLuo
authored
Merge pull request #1307 from taosdata/other/TD-12525
tdenginewrite rebuild
2 parents 53b2288 + 1a7a00c commit 9168f4c

File tree

66 files changed

+4257
-1541
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

66 files changed

+4257
-1541
lines changed

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
6262
| | Elasticsearch | | √ |[写](https://github.com/alibaba/DataX/blob/master/elasticsearchwriter/doc/elasticsearchwriter.md)|
6363
| 时间序列数据库 | OpenTSDB | √ | |[读](https://github.com/alibaba/DataX/blob/master/opentsdbreader/doc/opentsdbreader.md)|
6464
| | TSDB | √ | √ |[读](https://github.com/alibaba/DataX/blob/master/tsdbreader/doc/tsdbreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/tsdbwriter/doc/tsdbhttpwriter.md)|
65+
| | TDengine | √ | √ |[读](https://github.com/taosdata/DataX/blob/master/tdenginereader/doc/tdenginereader.md) 、[写](https://github.com/taosdata/DataX/blob/master/tdenginewriter/doc/tdenginewriter-CN.md)|
6566

6667
# 阿里云DataWorks数据集成
6768

‎package.xml‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,13 @@
180180
</includes>
181181
<outputDirectory>datax</outputDirectory>
182182
</fileSet>
183+
<fileSet>
184+
<directory>tdenginereader/target/datax/</directory>
185+
<includes>
186+
<include>**/*.*</include>
187+
</includes>
188+
<outputDirectory>datax</outputDirectory>
189+
</fileSet>
183190

184191
<!-- writer -->
185192
<fileSet>

‎pom.xml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,6 +108,7 @@
108108
<module>hbase20xsqlreader</module>
109109
<module>hbase20xsqlwriter</module>
110110
<module>kuduwriter</module>
111+
<module>tdenginereader</module>
111112
</modules>
112113

113114
<dependencyManagement>

‎tdenginereader/doc/tdenginereader-CN.md‎

Lines changed: 195 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,195 @@
1+
# DataX TDengineReader
2+
3+
## 1 快速介绍
4+
5+
TDengineReader 插件实现了 TDengine 读取数据的功能。
6+
7+
## 2 实现原理
8+
9+
TDengineReader 通过 TDengine 的 JDBC driver 查询获取数据。
10+
11+
## 3 功能说明
12+
13+
### 3.1 配置样例
14+
15+
* 配置一个从 TDengine 抽取数据作业:
16+
17+
```json
18+
{
19+
"job": {
20+
"content": [
21+
{
22+
"reader": {
23+
"name": "tdenginereader",
24+
"parameter": {
25+
"username": "root",
26+
"password": "taosdata",
27+
"connection": [
28+
{
29+
"table": [
30+
"meters"
31+
],
32+
"jdbcUrl": [
33+
"jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP"
34+
]
35+
}
36+
],
37+
"column": [
38+
"ts",
39+
"current",
40+
"voltage",
41+
"phase"
42+
],
43+
"where": "ts>=0",
44+
"beginDateTime": "2017-07-14 10:40:00",
45+
"endDateTime": "2017-08-14 10:40:00"
46+
}
47+
},
48+
"writer": {
49+
"name": "streamwriter",
50+
"parameter": {
51+
"encoding": "UTF-8",
52+
"print": true
53+
}
54+
}
55+
}
56+
],
57+
"setting": {
58+
"speed": {
59+
"channel": 1
60+
}
61+
}
62+
}
63+
}
64+
```
65+
66+
* 配置一个自定义 SQL 的数据抽取作业：
67+
68+
```json
69+
{
70+
"job": {
71+
"content": [
72+
{
73+
"reader": {
74+
"name": "tdenginereader",
75+
"parameter": {
76+
"user": "root",
77+
"password": "taosdata",
78+
"connection": [
79+
{
80+
"querySql": [
81+
"select * from test.meters"
82+
],
83+
"jdbcUrl": [
84+
"jdbc:TAOS-RS://192.168.56.105:6041/test?timestampFormat=TIMESTAMP"
85+
]
86+
}
87+
]
88+
}
89+
},
90+
"writer": {
91+
"name": "streamwriter",
92+
"parameter": {
93+
"encoding": "UTF-8",
94+
"print": true
95+
}
96+
}
97+
}
98+
],
99+
"setting": {
100+
"speed": {
101+
"channel": 1
102+
}
103+
}
104+
}
105+
}
106+
```
107+
108+
### 3.2 参数说明
109+
110+
* **username**
111+
* 描述：TDengine 实例的用户名 <br />
112+
* 必选：是 <br />
113+
* 默认值：无 <br />
114+
* **password**
115+
* 描述：TDengine 实例的密码 <br />
116+
* 必选：是 <br />
117+
* 默认值：无 <br />
118+
* **jdbcUrl**
119+
* 描述：TDengine 数据库的JDBC连接信息。注意，jdbcUrl必须包含在connection配置单元中。JdbcUrl具体请参看TDengine官方文档。
120+
* 必选：是 <br />
121+
* 默认值：无<br />
122+
* **querySql**
123+
* 描述：在有些业务场景下，where 这一配置项不足以描述所筛选的条件，用户可以通过该配置型来自定义筛选SQL。当用户配置了 querySql 后， TDengineReader 就会忽略 table, column,
124+
where, beginDateTime, endDateTime这些配置型，直接使用这个配置项的内容对数据进行筛选。例如需要 进行多表join后同步数据，使用 select a,b from table_a join
125+
table_b on table_a.id = table_b.id<br />
126+
* 必选：否 <br />
127+
* 默认值：无 <br />
128+
* **table**
129+
* 描述：所选取的需要同步的表。使用 JSON 的数组描述，因此支持多张表同时抽取。当配置为多张表时，用户自己需保证多张表是同一 schema 结构， TDengineReader不予检查表是否同一逻辑表。注意，table必须包含在
130+
connection 配置单元中。<br />
131+
* 必选：是 <br />
132+
* 默认值：无 <br />
133+
* **where**
134+
* 描述：筛选条件中的 where 子句，TDengineReader 根据指定的column, table, where, begingDateTime, endDateTime 条件拼接 SQL，并根据这个 SQL
135+
进行数据抽取。 <br />
136+
* 必选：否 <br />
137+
* 默认值：无 <br />
138+
* **beginDateTime**
139+
* 描述：数据的开始时间，Job 迁移从 begineDateTime 到 endDateTime 的数据，格式为 yyyy-MM-dd HH:mm:ss <br />
140+
* 必选：否 <br />
141+
* 默认值：无 <br />
142+
* **endDateTime**
143+
* 描述：数据的结束时间，Job 迁移从 begineDateTime 到 endDateTime 的数据，格式为 yyyy-MM-dd HH:mm:ss <br />
144+
* 必选：否 <br />
145+
* 默认值：无 <br />
146+
147+
### 3.3 类型转换
148+
149+
| TDengine 数据类型 | DataX 内部类型 |
150+
| --------------- | ------------- |
151+
| TINYINT | Long |
152+
| SMALLINT | Long |
153+
| INTEGER | Long |
154+
| BIGINT | Long |
155+
| FLOAT | Double |
156+
| DOUBLE | Double |
157+
| BOOLEAN | Bool |
158+
| TIMESTAMP | Date |
159+
| BINARY | Bytes |
160+
| NCHAR | String |
161+
162+
## 4 性能报告
163+
164+
### 4.1 环境准备
165+
166+
#### 4.1.1 数据特征
167+
168+
#### 4.1.2 机器参数
169+
170+
#### 4.1.3 DataX jvm 参数
171+
172+
-Xms1024m -Xmx1024m -XX:+HeapDumpOnOutOfMemoryError
173+
174+
### 4.2 测试报告
175+
176+
#### 4.2.1 单表测试报告
177+
178+
| 通道数| DataX速度(Rec/s)|DataX流量(MB/s)| DataX机器网卡流出流量(MB/s)|DataX机器运行负载|DB网卡进入流量(MB/s)|DB运行负载|DB TPS|
179+
|--------| --------|--------|--------|--------|--------|--------|--------|
180+
|1| | | | | | | |
181+
|4| | | | | | | |
182+
|8| | | | | | | |
183+
|16| | | | | | | |
184+
|32| | | | | | | |
185+
186+
说明：
187+
188+
#### 4.2.4 性能测试小结
189+
190+
1.
191+
2.
192+
193+
## 5 约束限制
194+
195+
## FAQ

‎tdenginereader/pom.xml‎

Lines changed: 135 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,135 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<project xmlns="http://maven.apache.org/POM/4.0.0"
3+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
4+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
5+
<parent>
6+
<artifactId>datax-all</artifactId>
7+
<groupId>com.alibaba.datax</groupId>
8+
<version>0.0.1-SNAPSHOT</version>
9+
</parent>
10+
<modelVersion>4.0.0</modelVersion>
11+
12+
<artifactId>tdenginereader</artifactId>
13+
14+
<properties>
15+
<maven.compiler.source>8</maven.compiler.source>
16+
<maven.compiler.target>8</maven.compiler.target>
17+
</properties>
18+
19+
<dependencies>
20+
<dependency>
21+
<groupId>com.alibaba.datax</groupId>
22+
<artifactId>datax-common</artifactId>
23+
<version>${datax-project-version}</version>
24+
<exclusions>
25+
<exclusion>
26+
<artifactId>slf4j-log4j12</artifactId>
27+
<groupId>org.slf4j</groupId>
28+
</exclusion>
29+
</exclusions>
30+
</dependency>
31+
32+
<dependency>
33+
<groupId>com.alibaba</groupId>
34+
<artifactId>fastjson</artifactId>
35+
<version>1.2.78</version>
36+
</dependency>
37+
38+
<dependency>
39+
<groupId>com.alibaba.datax.tdenginewriter</groupId>
40+
<artifactId>tdenginewriter</artifactId>
41+
<version>0.0.1-SNAPSHOT</version>
42+
<scope>compile</scope>
43+
</dependency>
44+
45+
<dependency>
46+
<groupId>com.taosdata.jdbc</groupId>
47+
<artifactId>taos-jdbcdriver</artifactId>
48+
<version>2.0.37</version>
49+
<exclusions>
50+
<exclusion>
51+
<groupId>com.alibaba</groupId>
52+
<artifactId>fastjson</artifactId>
53+
</exclusion>
54+
</exclusions>
55+
</dependency>
56+
57+
<dependency>
58+
<groupId>junit</groupId>
59+
<artifactId>junit</artifactId>
60+
<version>${junit-version}</version>
61+
<scope>test</scope>
62+
</dependency>
63+
<dependency>
64+
<groupId>com.alibaba.datax</groupId>
65+
<artifactId>plugin-rdbms-util</artifactId>
66+
<version>0.0.1-SNAPSHOT</version>
67+
<scope>compile</scope>
68+
</dependency>
69+
70+
<dependency>
71+
<groupId>com.alibaba.datax</groupId>
72+
<artifactId>datax-core</artifactId>
73+
<version>0.0.1-SNAPSHOT</version>
74+
<scope>test</scope>
75+
</dependency>
76+
<!-- 添加 dm8 jdbc jar 包依赖-->
77+
<!-- <dependency>-->
78+
<!-- <groupId>com.dameng</groupId>-->
79+
<!-- <artifactId>dm-jdbc</artifactId>-->
80+
<!-- <version>1.8</version>-->
81+
<!-- <scope>system</scope>-->
82+
<!-- <systemPath>${project.basedir}/src/test/resources/DmJdbcDriver18.jar</systemPath>-->
83+
<!-- </dependency>-->
84+
</dependencies>
85+
86+
<build>
87+
<plugins>
88+
<!-- compiler plugin -->
89+
<plugin>
90+
<artifactId>maven-compiler-plugin</artifactId>
91+
<configuration>
92+
<source>${jdk-version}</source>
93+
<target>${jdk-version}</target>
94+
<encoding>${project-sourceEncoding}</encoding>
95+
</configuration>
96+
</plugin>
97+
<plugin>
98+
<artifactId>maven-assembly-plugin</artifactId>
99+
<configuration>
100+
<descriptors>
101+
<descriptor>src/main/assembly/package.xml</descriptor>
102+
</descriptors>
103+
<finalName>datax</finalName>
104+
</configuration>
105+
<executions>
106+
<execution>
107+
<id>dwzip</id>
108+
<phase>package</phase>
109+
<goals>
110+
<goal>single</goal>
111+
</goals>
112+
</execution>
113+
</executions>
114+
</plugin>
115+
116+
<plugin>
117+
<groupId>org.apache.maven.plugins</groupId>
118+
<artifactId>maven-surefire-plugin</artifactId>
119+
<version>2.12.4</version>
120+
<configuration>
121+
<!-- 包含哪些测试用例 -->
122+
<includes>
123+
<include>**/*Test.java</include>
124+
</includes>
125+
<!-- 不包含哪些测试用例 -->
126+
<excludes>
127+
</excludes>
128+
<testFailureIgnore>true</testFailureIgnore>
129+
</configuration>
130+
</plugin>
131+
132+
</plugins>
133+
</build>
134+
135+
</project>

‎tdenginereader/src/main/assembly/package.xml‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"
2+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
3+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd">
4+
<id></id>
5+
<formats>
6+
<format>dir</format>
7+
</formats>
8+
<includeBaseDirectory>false</includeBaseDirectory>
9+
<fileSets>
10+
<fileSet>
11+
<directory>src/main/resources</directory>
12+
<includes>
13+
<include>plugin.json</include>
14+
<include>plugin_job_template.json</include>
15+
</includes>
16+
<outputDirectory>plugin/reader/tdenginereader</outputDirectory>
17+
</fileSet>
18+
<fileSet>
19+
<directory>target/</directory>
20+
<includes>
21+
<include>tdenginereader-0.0.1-SNAPSHOT.jar</include>
22+
</includes>
23+
<outputDirectory>plugin/reader/tdenginereader</outputDirectory>
24+
</fileSet>
25+
</fileSets>
26+
27+
<dependencySets>
28+
<dependencySet>
29+
<useProjectArtifact>false</useProjectArtifact>
30+
<outputDirectory>plugin/reader/tdenginereader/libs</outputDirectory>
31+
<scope>runtime</scope>
32+
</dependencySet>
33+
</dependencySets>
34+
</assembly>

0 commit comments

Comments
 (0)