Skip to content

Commit edce44e

Browse files
author
yud8
committed
[Feature-17501]Generic Third-party System API Connector for REST-based Task Scheduling #17501 
1 parent 9c6cf15 commit edce44e

File tree

71 files changed

+6105
-25
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+6105
-25
lines changed

‎config/plugins_config‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -108,4 +108,5 @@ dolphinscheduler-task-spark
108108
dolphinscheduler-task-sql
109109
dolphinscheduler-task-sqoop
110110
dolphinscheduler-task-zeppelin
111+
dolphinscheduler-task-external-system
111112
--end--

‎docs/img/tasks/icons/external_system.png‎

47.5 KB
Loading

‎dolphinscheduler-api/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -255,6 +255,11 @@
255255
<artifactId>azure-resourcemanager-datafactory</artifactId>
256256
</dependency>
257257

258+
<dependency>
259+
<groupId>com.jayway.jsonpath</groupId>
260+
<artifactId>json-path</artifactId>
261+
</dependency>
262+
258263
<dependency>
259264
<groupId>com.azure</groupId>
260265
<artifactId>azure-identity</artifactId>

‎dolphinscheduler-api/src/main/java/org/apache/dolphinscheduler/api/enums/Status.java‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,40 @@ public enum Status {
7373
VERIFY_DATASOURCE_NAME_FAILURE(10039, "verify datasource name failure", "验证数据源名称失败"),
7474
UNAUTHORIZED_DATASOURCE(10040, "unauthorized datasource", "未经授权的数据源"),
7575
AUTHORIZED_DATA_SOURCE(10041, "authorized data source", "授权数据源失败"),
76+
CREATE_EXTERNAL_SYSTEM_ERROR(11042, "create external system error", "创建第三方系统错误"),
77+
UPDATE_EXTERNAL_SYSTEM_ERROR(11043, "update external system error", "更新第三方系统错误"),
78+
QUERY_EXTERNAL_SYSTEM_ERROR(11044, "query external system error", "查询第三方系统错误"),
79+
DELETE_EXTERNAL_SYSTEM_ERROR(11045, "delete external system error", "删除第三方系统错误"),
80+
TEST_EXTERNAL_SYSTEM_CONNECTION_ERROR(11046, "connect external system failure", "建立第三方系统连接失败"),
81+
EXTERNAL_SYSTEM_NOT_EXIST(11047, "external system not exist", "第三方系统不存在"),
82+
EXTERNAL_SYSTEM_NAME_EXIST(11048, "external name exist", "第三方系统名称已存在"),
83+
EXTERNAL_SYSTEM_CONNECT_AUTH_FAILED(11049, "external connect failed", "第三方系统连接失败,检查认证信息"),
84+
EXTERNAL_SYSTEM_CONNECT_SELECT_FAILED(11049, "external connect failed", "第三方系统连接失败，检查查询任务列表接口信息"),
85+
UNAUTHORIZED_EXTERNAL_SYSTEM(10050, "unauthorized datasource", "未经授权的第三方系统"),
86+
AUTHORIZED_EXTERNAL_SYSTEM(10051, "authorized data source", "授权第三方系统失败"),
87+
EXTERNAL_SYSTEM_NAME_EMPTY(11052, "external system name is empty", "第三方系统名称为空"),
88+
EXTERNAL_SYSTEM_SERVICE_ADDRESS_EMPTY(11053, "external system service address is empty", "第三方系统服务地址为空"),
89+
EXTERNAL_SYSTEM_AUTH_CONFIG_EMPTY(11054, "external system auth config is empty", "第三方系统认证配置为空"),
90+
EXTERNAL_SYSTEM_AUTH_CONFIG_TYPE_EMPTY(11055, "external system auth config type is empty", "第三方系统认证类型为空"),
91+
EXTERNAL_SYSTEM_BASIC_USERNAME_EMPTY(11056, "external system basic username is empty", "第三方系统基础认证用户名为空"),
92+
EXTERNAL_SYSTEM_BASIC_PASSWORD_EMPTY(11057, "external system basic password is empty", "第三方系统基础认证密码为空"),
93+
EXTERNAL_SYSTEM_JWT_TOKEN_EMPTY(11058, "external system jwt token is empty", "第三方系统JWT令牌为空"),
94+
EXTERNAL_SYSTEM_OAUTH2_TOKEN_URL_EMPTY(11059, "external system oauth2 token url is empty", "第三方系统OAuth2令牌URL为空"),
95+
EXTERNAL_SYSTEM_OAUTH2_CLIENT_ID_EMPTY(11060, "external system oauth2 client id is empty", "第三方系统OAuth2客户端ID为空"),
96+
EXTERNAL_SYSTEM_OAUTH2_CLIENT_SECRET_EMPTY(11061, "external system oauth2 client secret is empty",
97+
"第三方系统OAuth2客户端密钥为空"),
98+
EXTERNAL_SYSTEM_OAUTH2_GRANT_TYPE_EMPTY(11062, "external system oauth2 grant type is empty", "第三方系统OAuth2授权类型为空"),
99+
EXTERNAL_SYSTEM_OAUTH2_USERNAME_EMPTY(11063, "external system oauth2 username is empty", "第三方系统OAuth2用户名为空"),
100+
EXTERNAL_SYSTEM_OAUTH2_PASSWORD_EMPTY(11064, "external system oauth2 password is empty", "第三方系统OAuth2密码为空"),
101+
EXTERNAL_SYSTEM_AUTH_TYPE_UNSUPPORTED(11065, "external system auth type is unsupported", "第三方系统认证类型不支持"),
102+
EXTERNAL_SYSTEM_SELECT_INTERFACE_EMPTY(11066, "external system select interface is empty", "第三方系统查询接口配置为空"),
103+
EXTERNAL_SYSTEM_SUBMIT_INTERFACE_EMPTY(11067, "external system submit interface is empty", "第三方系统提交接口配置为空"),
104+
EXTERNAL_SYSTEM_POLL_STATUS_INTERFACE_EMPTY(11068, "external system poll status interface is empty",
105+
"第三方系统轮询状态接口配置为空"),
106+
EXTERNAL_SYSTEM_STOP_INTERFACE_EMPTY(11069, "external system stop interface is empty", "第三方系统停止接口配置为空"),
107+
EXTERNAL_SYSTEM_INTERFACE_URL_EMPTY(11070, "external system interface url is empty", "第三方系统接口URL为空"),
108+
EXTERNAL_SYSTEM_INTERFACE_METHOD_EMPTY(11071, "external system interface method is empty", "第三方系统接口方法为空"),
109+
EXTERNAL_SYSTEM_NAME_TOO_LONG(11071, "external system name too long", "第三方系统接口名称太长"),
76110
LOGIN_SUCCESS(10042, "login success", "登录成功"),
77111
USER_LOGIN_FAILURE(10043, "user login failure", "用户登录失败"),
78112
LIST_WORKERS_ERROR(10044, "list workers error", "查询worker列表错误"),
@@ -120,6 +154,7 @@ public enum Status {
120154
GRANT_PROJECT_ERROR(10094, "grant project error", "授权项目错误"),
121155
GRANT_RESOURCE_ERROR(10095, "grant resource error", "授权资源错误"),
122156
GRANT_DATASOURCE_ERROR(10097, "grant datasource error", "授权数据源错误"),
157+
GRANT_EXTERNALSYSTEM_ERROR(100971, "grant external system error", "授权第三方系统错误"),
123158
GET_USER_INFO_ERROR(10098, "get user info error", "获取用户信息错误"),
124159
USER_LIST_ERROR(10099, "user list error", "查询用户列表错误"),
125160
VERIFY_USERNAME_ERROR(10100, "verify username error", "用户名验证错误"),

‎dolphinscheduler-api/src/main/resources/task-type-config.yaml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ task:
3131
- 'FLINK_STREAM'
3232
- 'HIVECLI'
3333
- 'REMOTESHELL'
34+
- 'EXTERNAL_SYSTEM'
3435
cloud:
3536
- 'EMR'
3637
- 'K8S'

‎dolphinscheduler-bom/pom.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,7 @@
124124
<zeppelin-client.version>0.10.1</zeppelin-client.version>
125125
<aliyun-voice.version>2.1.4</aliyun-voice.version>
126126
<nashorn-sandbox.version>0.3.2</nashorn-sandbox.version>
127+
<io-jsonwebtoken.version>0.11.5</io-jsonwebtoken.version>
127128
<brace-expansion.version>2.0.1</brace-expansion.version>
128129
<protoc.version>3.17.3</protoc.version>
129130
<j2objc.version>1.3</j2objc.version>
@@ -1038,6 +1039,23 @@
10381039
<artifactId>jdbc</artifactId>
10391040
<version>${dolphindb-jdbc.version}</version>
10401041
</dependency>
1042+
1043+
<dependency>
1044+
<groupId>io.jsonwebtoken</groupId>
1045+
<artifactId>jjwt-api</artifactId>
1046+
<version>${io-jsonwebtoken.version}</version>
1047+
</dependency>
1048+
<dependency>
1049+
<groupId>io.jsonwebtoken</groupId>
1050+
<artifactId>jjwt-impl</artifactId>
1051+
<version>${io-jsonwebtoken.version}</version>
1052+
</dependency>
1053+
<dependency>
1054+
<groupId>io.jsonwebtoken</groupId>
1055+
<artifactId>jjwt-jackson</artifactId>
1056+
<version>${io-jsonwebtoken.version}</version>
1057+
</dependency>
1058+
10411059
</dependencies>
10421060
</dependencyManagement>
10431061

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/constants/Constants.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -212,6 +212,11 @@ public final class Constants {
212212

213213
public static final String DEFAULT = "default";
214214
public static final String PASSWORD = "password";
215+
public static final String BASICPASSWORD = "basicPassword";
216+
public static final String OAUTH2CLIENTSECRET = "oauth2ClientSecret";
217+
public static final String JWTTOKEN = "jwtToken";
218+
public static final String OAUTH2PASSWORD = "oauth2Password";
219+
215220
public static final String XXXXXX = "******";
216221
public static final String NULL = "NULL";
217222
public static final String THREAD_NAME_MASTER_SERVER = "Master-Server";

‎dolphinscheduler-common/src/main/java/org/apache/dolphinscheduler/common/utils/OkHttpUtils.java‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -99,6 +99,27 @@ public class OkHttpUtils {
9999
}
100100
}
101101

102+
public static @NonNull OkHttpResponse postFormBody(@NonNull String url,
103+
@Nullable OkHttpRequestHeaders okHttpRequestHeaders,
104+
@Nullable Map<String, Object> requestParamsMap,
105+
@Nullable RequestBody formBody,
106+
int connectTimeout,
107+
int writeTimeout,
108+
int readTimeout) throws IOException {
109+
OkHttpClient client = getHttpClient(connectTimeout, writeTimeout, readTimeout);
110+
String finalUrl = addUrlParams(requestParamsMap, url);
111+
Request.Builder requestBuilder = new Request.Builder().url(finalUrl);
112+
addHeader(okHttpRequestHeaders.getHeaders(), requestBuilder);
113+
Request request = requestBuilder
114+
.post(formBody) // 明确使用POST方法
115+
.build();
116+
try (Response response = client.newCall(request).execute()) {
117+
return new OkHttpResponse(response.code(), getResponseBody(response));
118+
} catch (Exception e) {
119+
throw new RuntimeException(String.format("Post request execute failed, url: %s", url), e);
120+
}
121+
}
122+
102123
/**
103124
* http put request
104125
* @param connectTimeout connect timeout in milliseconds

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-all/pom.xml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,5 +168,10 @@
168168
<artifactId>dolphinscheduler-datasource-dolphindb</artifactId>
169169
<version>${project.version}</version>
170170
</dependency>
171+
<dependency>
172+
<groupId>org.apache.dolphinscheduler</groupId>
173+
<artifactId>dolphinscheduler-datasource-thirdpartysystemconnector</artifactId>
174+
<version>${project.version}</version>
175+
</dependency>
171176
</dependencies>
172177
</project>

‎dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-thirdpartysystemconnector/pom.xml‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Licensed to the Apache Software Foundation (ASF) under one or more
4+
~ contributor license agreements. See the NOTICE file distributed with
5+
~ this work for additional information regarding copyright ownership.
6+
~ The ASF licenses this file to You under the Apache License, Version 2.0
7+
~ (the "License"); you may not use this file except in compliance with
8+
~ the License. You may obtain a copy of the License at
9+
~
10+
~ http://www.apache.org/licenses/LICENSE-2.0
11+
~
12+
~ Unless required by applicable law or agreed to in writing, software
13+
~ distributed under the License is distributed on an "AS IS" BASIS,
14+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
~ See the License for the specific language governing permissions and
16+
~ limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.dolphinscheduler</groupId>
23+
<artifactId>dolphinscheduler-datasource-plugin</artifactId>
24+
<version>dev-SNAPSHOT</version>
25+
</parent>
26+
27+
<artifactId>dolphinscheduler-datasource-thirdpartysystemconnector</artifactId>
28+
<packaging>jar</packaging>
29+
<name>${project.artifactId}</name>
30+
31+
<properties>
32+
<datasource.plugin.name>thirdpartysystemconnector</datasource.plugin.name>
33+
</properties>
34+
35+
<dependencies>
36+
<dependency>
37+
<groupId>org.apache.dolphinscheduler</groupId>
38+
<artifactId>dolphinscheduler-datasource-api</artifactId>
39+
<scope>provided</scope>
40+
</dependency>
41+
42+
<dependency>
43+
<groupId>org.apache.dolphinscheduler</groupId>
44+
<artifactId>dolphinscheduler-common</artifactId>
45+
<scope>provided</scope>
46+
</dependency>
47+
48+
<dependency>
49+
<groupId>org.apache.dolphinscheduler</groupId>
50+
<artifactId>dolphinscheduler-spi</artifactId>
51+
<scope>provided</scope>
52+
</dependency>
53+
</dependencies>
54+
55+
<build>
56+
<plugins>
57+
<plugin>
58+
<groupId>org.apache.maven.plugins</groupId>
59+
<artifactId>maven-shade-plugin</artifactId>
60+
<executions>
61+
<execution>
62+
<goals>
63+
<goal>shade</goal>
64+
</goals>
65+
<phase>package</phase>
66+
</execution>
67+
</executions>
68+
</plugin>
69+
</plugins>
70+
</build>
71+
</project>

0 commit comments

Comments
 (0)