Skip to content

Commit 2f5c9cf

Browse files
TrafalgarLuoTrafalgarLuo
authored
Merge pull request #1726 from alibaba/opensource_2303
Opensource 2303
2 parents f01836a + 85c2211 commit 2f5c9cf

File tree

21 files changed

+24
-327
lines changed

21 files changed

+24
-327
lines changed

‎README.md‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ DataX本身作为数据同步框架，将不同数据源的同步抽象为从源
2626

2727
# Quick Start
2828

29-
##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202210/datax.tar.gz)
29+
##### Download [DataX下载地址](https://datax-opensource.oss-cn-hangzhou.aliyuncs.com/202303/datax.tar.gz)
3030

3131

3232
##### 请点击：[Quick Start](https://github.com/alibaba/DataX/blob/master/userGuid.md)
@@ -70,6 +70,7 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
7070
| | Databend | | √ | [写](https://github.com/alibaba/DataX/blob/master/databendwriter/doc/databendwriter.md) |
7171
| | Hive | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md) |
7272
| | kudu | | √ | [写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md) |
73+
| | selectdb | | √ | [写](https://github.com/alibaba/DataX/blob/master/selectdbwriter/doc/selectdbwriter.md) |
7374
| 无结构化数据存储 | TxtFile | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/txtfilereader/doc/txtfilereader.md) 、[写](https://github.com/alibaba/DataX/blob/master/txtfilewriter/doc/txtfilewriter.md) |
7475
| | FTP | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/ftpreader/doc/ftpreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/ftpwriter/doc/ftpwriter.md) |
7576
| | HDFS | √ | √ | [读](https://github.com/alibaba/DataX/blob/master/hdfsreader/doc/hdfsreader.md) 、[写](https://github.com/alibaba/DataX/blob/master/hdfswriter/doc/hdfswriter.md) |
@@ -108,6 +109,12 @@ DataX目前已经有了比较全面的插件体系，主流的RDBMS数据库、N
108109

109110
DataX 后续计划月度迭代更新，也欢迎感兴趣的同学提交 Pull requests，月度更新内容会介绍介绍如下。
110111

112+
- [datax_v202303]（https://github.com/alibaba/DataX/releases/tag/datax_v202303)
113+
- 精简代码
114+
- 新增插件（adbmysqlwriter、databendwriter、selectdbwriter）
115+
- 优化插件、修复问题（sqlserver、hdfs、cassandra、kudu、oss）
116+
- fastjson 升级到 fastjson2
117+
111118
- [datax_v202210]（https://github.com/alibaba/DataX/releases/tag/datax_v202210)
112119
- 涉及通道能力更新（OceanBase、Tdengine、Doris等）
113120

‎adswriter/doc/adswriter.md‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,6 @@ DataX 将数据直连ADS接口，利用ADS暴露的INSERT接口直写到ADS。
110110
"account": "xxx@aliyun.com",
111111
"odpsServer": "xxx",
112112
"tunnelServer": "xxx",
113-
"accountType": "aliyun",
114113
"project": "transfer_project"
115114
},
116115
"writeMode": "load",

‎adswriter/src/main/java/com/alibaba/datax/plugin/writer/adswriter/load/TransferProjectConf.java‎

Lines changed: 0 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -12,15 +12,13 @@ public class TransferProjectConf {
1212
public final static String KEY_ACCOUNT = "odps.account";
1313
public final static String KEY_ODPS_SERVER = "odps.odpsServer";
1414
public final static String KEY_ODPS_TUNNEL = "odps.tunnelServer";
15-
public final static String KEY_ACCOUNT_TYPE = "odps.accountType";
1615
public final static String KEY_PROJECT = "odps.project";
1716

1817
private String accessId;
1918
private String accessKey;
2019
private String account;
2120
private String odpsServer;
2221
private String odpsTunnel;
23-
private String accountType;
2422
private String project;
2523

2624
public static TransferProjectConf create(Configuration adsWriterConf) {
@@ -30,7 +28,6 @@ public static TransferProjectConf create(Configuration adsWriterConf) {
3028
res.account = adsWriterConf.getString(KEY_ACCOUNT);
3129
res.odpsServer = adsWriterConf.getString(KEY_ODPS_SERVER);
3230
res.odpsTunnel = adsWriterConf.getString(KEY_ODPS_TUNNEL);
33-
res.accountType = adsWriterConf.getString(KEY_ACCOUNT_TYPE, "aliyun");
3431
res.project = adsWriterConf.getString(KEY_PROJECT);
3532
return res;
3633
}
@@ -55,9 +52,6 @@ public String getOdpsTunnel() {
5552
return odpsTunnel;
5653
}
5754

58-
public String getAccountType() {
59-
return accountType;
60-
}
6155

6256
public String getProject() {
6357
return project;

‎common/src/main/java/com/alibaba/datax/common/statistics/PerfTrace.java‎

Lines changed: 5 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@ public class PerfTrace {
3131
private int taskGroupId;
3232
private int channelNumber;
3333

34-
private int priority;
3534
private int batchSize = 500;
3635
private volatile boolean perfReportEnable = true;
3736

@@ -54,12 +53,12 @@ public class PerfTrace {
5453
* @param taskGroupId
5554
* @return
5655
*/
57-
public static PerfTrace getInstance(boolean isJob, long jobId, int taskGroupId, int priority, boolean enable) {
56+
public static PerfTrace getInstance(boolean isJob, long jobId, int taskGroupId, boolean enable) {
5857

5958
if (instance == null) {
6059
synchronized (lock) {
6160
if (instance == null) {
62-
instance = new PerfTrace(isJob, jobId, taskGroupId, priority, enable);
61+
instance = new PerfTrace(isJob, jobId, taskGroupId, enable);
6362
}
6463
}
6564
}
@@ -76,22 +75,21 @@ public static PerfTrace getInstance() {
7675
LOG.error("PerfTrace instance not be init! must have some error! ");
7776
synchronized (lock) {
7877
if (instance == null) {
79-
instance = new PerfTrace(false, -1111, -1111, 0, false);
78+
instance = new PerfTrace(false, -1111, -1111, false);
8079
}
8180
}
8281
}
8382
return instance;
8483
}
8584

86-
private PerfTrace(boolean isJob, long jobId, int taskGroupId, int priority, boolean enable) {
85+
private PerfTrace(boolean isJob, long jobId, int taskGroupId, boolean enable) {
8786
try {
8887
this.perfTraceId = isJob ? "job_" + jobId : String.format("taskGroup_%s_%s", jobId, taskGroupId);
8988
this.enable = enable;
9089
this.isJob = isJob;
9190
this.taskGroupId = taskGroupId;
9291
this.instId = jobId;
93-
this.priority = priority;
94-
LOG.info(String.format("PerfTrace traceId=%s, isEnable=%s, priority=%s", this.perfTraceId, this.enable, this.priority));
92+
LOG.info(String.format("PerfTrace traceId=%s, isEnable=%s", this.perfTraceId, this.enable));
9593

9694
} catch (Exception e) {
9795
// do nothing
@@ -398,7 +396,6 @@ public synchronized JobStatisticsDto2 getReports(String mode) {
398396
jdo.setWindowEnd(this.windowEnd);
399397
jdo.setJobStartTime(jobStartTime);
400398
jdo.setJobRunTimeMs(System.currentTimeMillis() - jobStartTime.getTime());
401-
jdo.setJobPriority(this.priority);
402399
jdo.setChannelNum(this.channelNumber);
403400
jdo.setCluster(this.cluster);
404401
jdo.setJobDomain(this.jobDomain);
@@ -609,7 +606,6 @@ class JobStatisticsDto2 {
609606
private Date jobStartTime;
610607
private Date jobEndTime;
611608
private Long jobRunTimeMs;
612-
private Integer jobPriority;
613609
private Integer channelNum;
614610
private String cluster;
615611
private String jobDomain;
@@ -680,10 +676,6 @@ public Long getJobRunTimeMs() {
680676
return jobRunTimeMs;
681677
}
682678

683-
public Integer getJobPriority() {
684-
return jobPriority;
685-
}
686-
687679
public Integer getChannelNum() {
688680
return channelNum;
689681
}
@@ -816,10 +808,6 @@ public void setJobRunTimeMs(Long jobRunTimeMs) {
816808
this.jobRunTimeMs = jobRunTimeMs;
817809
}
818810

819-
public void setJobPriority(Integer jobPriority) {
820-
this.jobPriority = jobPriority;
821-
}
822-
823811
public void setChannelNum(Integer channelNum) {
824812
this.channelNum = channelNum;
825813
}

‎common/src/main/java/com/alibaba/datax/common/util/IdAndKeyRollingUtil.java‎

Lines changed: 0 additions & 62 deletions
This file was deleted.

‎core/src/main/java/com/alibaba/datax/core/Engine.java‎

Lines changed: 1 addition & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -79,16 +79,9 @@ public void start(Configuration allConf) {
7979
perfReportEnable = false;
8080
}
8181

82-
int priority = 0;
83-
try {
84-
priority = Integer.parseInt(System.getenv("SKYNET_PRIORITY"));
85-
}catch (NumberFormatException e){
86-
LOG.warn("prioriy set to 0, because NumberFormatException, the value is: "+System.getProperty("PROIORY"));
87-
}
88-
8982
Configuration jobInfoConfig = allConf.getConfiguration(CoreConstant.DATAX_JOB_JOBINFO);
9083
//初始化PerfTrace
91-
PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, priority, traceEnable);
84+
PerfTrace perfTrace = PerfTrace.getInstance(isJob, instanceId, taskGroupId, traceEnable);
9285
perfTrace.setJobInfo(jobInfoConfig,perfReportEnable,channelNumber);
9386
container.start();
9487

‎kuduwriter/src/main/java/com/q1/datax/plugin/writer/kudu11xwriter/KuduWriterTask.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ public void startWriter(RecordReceiver lineReceiver, TaskPluginCollector taskPlu
134134
break;
135135
case BOOLEAN:
136136
synchronized (lock) {
137-
row.addBoolean(name, Boolean.getBoolean(rawData));
137+
row.addBoolean(name, Boolean.parseBoolean(rawData));
138138
}
139139
break;
140140
case STRING:

‎mongodbreader/doc/mongodbreader.md‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -114,8 +114,7 @@ MongoDBReader通过Datax框架从MongoDB并行的读取数据，通过主控的J
114114
"accessKey": "********************",
115115
"truncate": true,
116116
"odpsServer": "xxx/api",
117-
"tunnelServer": "xxx",
118-
"accountType": "aliyun"
117+
"tunnelServer": "xxx"
119118
}
120119
}
121120
}

‎odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/Constant.java‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -14,20 +14,9 @@ public class Constant {
1414

1515
public static final String PARTITION_SPLIT_MODE = "partition";
1616

17-
public static final String DEFAULT_ACCOUNT_TYPE = "aliyun";
18-
19-
public static final String TAOBAO_ACCOUNT_TYPE = "taobao";
20-
2117
// 常量字段用COLUMN_CONSTANT_FLAG 首尾包住即可
2218
public final static String COLUMN_CONSTANT_FLAG = "'";
2319

24-
/**
25-
* 以下是获取accesskey id 需要用到的常量值
26-
*/
27-
public static final String SKYNET_ACCESSID = "SKYNET_ACCESSID";
28-
29-
public static final String SKYNET_ACCESSKEY = "SKYNET_ACCESSKEY";
30-
3120
public static final String PARTITION_COLUMNS = "partitionColumns";
3221

3322
public static final String PARSED_COLUMNS = "parsedColumns";

‎odpsreader/src/main/java/com/alibaba/datax/plugin/reader/odpsreader/Key.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,9 +24,6 @@ public class Key {
2424
// 当值为：partition 则只切分到分区；当值为：record，则当按照分区切分后达不到adviceNum时，继续按照record切分
2525
public final static String SPLIT_MODE = "splitMode";
2626

27-
// 账号类型，默认为aliyun，也可能为taobao等其他类型
28-
public final static String ACCOUNT_TYPE = "accountType";
29-
3027
public final static String PACKAGE_AUTHORIZED_PROJECT = "packageAuthorizedProject";
3128

3229
public final static String IS_COMPRESS = "isCompress";

0 commit comments

Comments
 (0)