Skip to content

Commit 547290c

Browse files
zqburdezqburde
authored
Merge pull request #996 from WeDataSphere/master
Fix problems in 1.1.0.
2 parents adca574 + f5cde09 commit 547290c

File tree

100 files changed

+258
-995
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

100 files changed

+258
-995
lines changed

‎.github/workflows/dlc.json‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,9 @@
11
{
2+
"ignorePatterns": [
3+
{
4+
"pattern": "https://github.com/WeBankFinTech/DataSphereStudio-Doc"
5+
}
6+
],
27
"timeout": "10s",
38
"retryOn429": true,
49
"retryCount": 10,
@@ -8,4 +13,5 @@
813
401,
914
403
1015
]
11-
}
16+
}
17+

‎dss-appconn/appconns/dss-dolphinscheduler-appconn/src/main/resources/init.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ VALUES ('dolphinscheduler', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appco
99
select @dolphinscheduler_appconnId:=id from `dss_appconn` where `appconn_name` = 'dolphinscheduler';
1010

1111
insert into `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`)
12-
values(@dolphinscheduler_appconnId,'DEV','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/','','dolphinscheduler');
12+
values(@dolphinscheduler_appconnId,'DEV','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/','{"reqUri":"dolphinscheduler/projects/analysis/define-user-count"}','dolphinscheduler');
1313

1414
-- 看appconn组件是要归属于哪个菜单
1515
select @dolphinscheduler_menuId:=id from dss_workspace_menu where name = "生产运维";

‎dss-appconn/appconns/dss-schedulis-appconn/src/main/resources/init.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ VALUES ('schedulis', 0, 1, 1, 1, NULL, 'com.webank.wedatasphere.dss.appconn.sche
99
select @schedulis_appconnId:=id from `dss_appconn` where `appconn_name` = 'schedulis';
1010

1111
insert into `dss_appconn_instance` (`appconn_id`, `label`, `url`, `enhance_json`, `homepage_uri`)
12-
values(@schedulis_appconnId,'DEV','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/','','');
12+
values(@schedulis_appconnId,'DEV','http://APPCONN_INSTALL_IP:APPCONN_INSTALL_PORT/','{"reqUri":""}','/manager');
1313

1414
-- 看appconn组件是要归属于哪个菜单
1515
select @schedulis_menuId:=id from dss_workspace_menu where name = "生产运维";

‎dss-framework/dss-framework-orchestrator-server/pom.xml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -149,6 +149,22 @@
149149
<version>${dss.version}</version>
150150
<scope>provided</scope>
151151
</dependency>
152+
<dependency>
153+
<groupId>org.apache.linkis</groupId>
154+
<artifactId>linkis-storage-script-dev-server</artifactId>
155+
<version>${linkis.version}</version>
156+
<exclusions>
157+
<exclusion>
158+
<artifactId>spring-context-support</artifactId>
159+
<groupId>org.springframework</groupId>
160+
</exclusion>
161+
</exclusions>
162+
</dependency>
163+
<dependency>
164+
<groupId>org.apache.linkis</groupId>
165+
<artifactId>linkis-storage</artifactId>
166+
<version>${linkis.version}</version>
167+
</dependency>
152168
</dependencies>
153169

154170
<build>

‎dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/restful/OrchestratorIERestful.java‎

Lines changed: 39 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
package com.webank.wedatasphere.dss.orchestrator.server.restful;
1818

1919
import com.webank.wedatasphere.dss.common.exception.DSSErrorException;
20+
import com.webank.wedatasphere.dss.common.exception.DSSRuntimeException;
2021
import com.webank.wedatasphere.dss.common.label.DSSLabel;
2122
import com.webank.wedatasphere.dss.common.label.EnvDSSLabel;
2223
import com.webank.wedatasphere.dss.common.label.LabelKeyConvertor;
@@ -31,8 +32,11 @@
3132
import com.webank.wedatasphere.dss.standard.app.sso.Workspace;
3233
import com.webank.wedatasphere.dss.standard.sso.utils.SSOHelper;
3334
import org.apache.commons.io.IOUtils;
35+
import org.apache.linkis.common.io.FsPath;
36+
import org.apache.linkis.filesystem.service.FsService;
3437
import org.apache.linkis.server.Message;
3538
import org.apache.linkis.server.security.SecurityFilter;
39+
import org.apache.linkis.storage.fs.FileSystem;
3640
import org.slf4j.Logger;
3741
import org.slf4j.LoggerFactory;
3842
import org.springframework.beans.factory.annotation.Autowired;
@@ -57,6 +61,8 @@ public class OrchestratorIERestful {
5761
@Autowired
5862
private BMLService bmlService;
5963
@Autowired
64+
private FsService fsService;
65+
@Autowired
6066
OrchestratorService orchestratorService;
6167
@Autowired
6268
private DSSOrchestratorContext orchestratorContext;
@@ -66,30 +72,43 @@ public Message importOrcFile(HttpServletRequest req,
6672
@RequestParam(required = false, name = "projectName") String projectName,
6773
@RequestParam(required = false, name = "projectID") Long projectID,
6874
@RequestParam(required = false, name = "labels") String labels,
69-
@RequestParam(required = false, name = "file") List<MultipartFile> files) throws DSSErrorException, Exception {
70-
if (null == files || files.size() == 0) {
75+
@RequestParam(required = false, name = "packageFile") MultipartFile packageFile,
76+
@RequestParam(required = false, name = "packageUri") String packageUri) throws Exception {
77+
78+
if (null == packageFile && packageUri == null) {
7179
throw new DSSErrorException(100788, "Import orchestrator failed for files is empty");
7280
}
73-
Long importOrcId = 0L;
74-
for (MultipartFile p : files) {
75-
InputStream inputStream = p.getInputStream();
76-
String fileName = new String(p.getOriginalFilename().getBytes("ISO8859-1"), "UTF-8");
77-
String userName = SecurityFilter.getLoginUsername(req);
78-
//调用工具类生产label
79-
List<DSSLabel> dssLabelList = getDSSLabelList(labels);
80-
Workspace workspace = SSOHelper.getWorkspace(req);
81+
String userName = SecurityFilter.getLoginUsername(req);
82+
//调用工具类生产label
83+
List<DSSLabel> dssLabelList = getDSSLabelList(labels);
84+
Workspace workspace = SSOHelper.getWorkspace(req);
85+
InputStream inputStream;
86+
String fileName;
87+
if (packageFile != null) {
88+
inputStream = packageFile.getInputStream();
89+
fileName = new String(packageFile.getOriginalFilename().getBytes("ISO8859-1"), "UTF-8");
8190
//3、打包新的zip包上传BML
82-
Map<String, Object> resultMap = bmlService.upload(userName, inputStream,
83-
fileName, projectName);
84-
try {
85-
RequestImportOrchestrator importRequest = new RequestImportOrchestrator(userName, projectName,
86-
projectID, resultMap.get("resourceId").toString(),
87-
resultMap.get("version").toString(), null, dssLabelList, workspace);
88-
importOrcId = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestrator(importRequest);
89-
} catch (Exception e) {
90-
logger.error("Import orchestrator failed for ", e);
91-
throw new DSSErrorException(100789, "Import orchestrator failed for " + e.getMessage());
91+
logger.info("User {} begin to import orchestrator file", userName);
92+
} else {
93+
FsPath fsPath = new FsPath(packageUri);
94+
FileSystem fileSystem = fsService.getFileSystem(userName, fsPath);
95+
if ( !fileSystem.exists(fsPath) ) {
96+
throw new DSSRuntimeException("路径上不存在文件！");
9297
}
98+
inputStream = fileSystem.read(fsPath);
99+
fileName = packageUri.substring(packageUri.lastIndexOf('/') + 1);
100+
}
101+
102+
Map<String, Object> resultMap = bmlService.upload(userName, inputStream, fileName, projectName);
103+
Long importOrcId;
104+
try {
105+
RequestImportOrchestrator importRequest = new RequestImportOrchestrator(userName, projectName,
106+
projectID, resultMap.get("resourceId").toString(),
107+
resultMap.get("version").toString(), null, dssLabelList, workspace);
108+
importOrcId = orchestratorContext.getDSSOrchestratorPlugin(ImportDSSOrchestratorPlugin.class).importOrchestrator(importRequest);
109+
} catch (Exception e) {
110+
logger.error("Import orchestrator failed for ", e);
111+
throw new DSSErrorException(100789, "Import orchestrator failed for " + e.getMessage());
93112
}
94113
return Message.ok().data("importOrcId", importOrcId);
95114
}

‎dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorFrameworkServiceImpl.java‎

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.webank.wedatasphere.dss.orchestrator.server.service.impl;
1818

19+
import com.webank.wedatasphere.dss.appconn.core.AppConn;
1920
import com.webank.wedatasphere.dss.appconn.scheduler.SchedulerAppConn;
2021
import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationCreationOperation;
2122
import com.webank.wedatasphere.dss.appconn.scheduler.structure.orchestration.OrchestrationDeletionOperation;
@@ -41,6 +42,7 @@
4142
import com.webank.wedatasphere.dss.orchestrator.core.type.DSSOrchestratorRelationManager;
4243
import com.webank.wedatasphere.dss.orchestrator.core.utils.OrchestratorUtils;
4344
import com.webank.wedatasphere.dss.orchestrator.db.dao.OrchestratorMapper;
45+
import com.webank.wedatasphere.dss.orchestrator.loader.LinkedAppConnResolver;
4446
import com.webank.wedatasphere.dss.orchestrator.loader.OrchestratorManager;
4547
import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorCreateRequest;
4648
import com.webank.wedatasphere.dss.orchestrator.server.entity.request.OrchestratorDeleteRequest;
@@ -64,6 +66,7 @@
6466
import java.util.Collections;
6567
import java.util.Date;
6668
import java.util.List;
69+
import java.util.Optional;
6770
import java.util.function.BiFunction;
6871
import java.util.function.Function;
6972
import java.util.stream.Collectors;
@@ -79,7 +82,8 @@ public class OrchestratorFrameworkServiceImpl implements OrchestratorFrameworkSe
7982
private OrchestratorService newOrchestratorService;
8083
@Autowired
8184
private OrchestratorManager orchestratorManager;
82-
85+
@Autowired
86+
private LinkedAppConnResolver linkedAppConnResolver;
8387
/**
8488
* 1.拿到的dss orchestrator的appconn
8589
* 2.然后创建
@@ -145,7 +149,7 @@ private <K extends StructureRequestRef, V extends ResponseRef> V tryOrchestratio
145149
Long refProjectId, refOrchestrationId;
146150
if (askProjectSender) {
147151
ProjectRefIdResponse projectRefIdResponse = (ProjectRefIdResponse) DSSSenderServiceFactory.getOrCreateServiceInstance().getProjectServerSender()
148-
.ask(new ProjectRefIdRequest(orchestrationPair.getValue().getId(), dssOrchestrator.getProjectId()));
152+
.ask(new ProjectRefIdRequest(Optional.ofNullable(orchestrationPair).map(ImmutablePair::getValue).map(AppInstance::getId).orElse(null), dssOrchestrator.getProjectId()));
149153
refProjectId = projectRefIdResponse.getRefProjectId();
150154
refOrchestrationId = null;
151155
} else {
@@ -244,11 +248,16 @@ protected ImmutablePair<OrchestrationService, AppInstance> getOrchestrationServi
244248
dssOrchestratorInfo.setLinkedAppConnNames(dssOrchestrator.getLinkedAppConn().stream().map(appConn -> appConn.getAppDesc().getAppName()).collect(Collectors.toList()));
245249
}
246250
SchedulerAppConn appConn = dssOrchestrator.getSchedulerAppConn();
247-
if (appConn == null) {
248-
throw new ExternalOperationWarnException(50322, "DSSOrchestrator " + dssOrchestrator.getName() + " has no SchedulerAppConn.");
251+
// if (appConn == null) {
252+
// throw new ExternalOperationWarnException(50322, "DSSOrchestrator " + dssOrchestrator.getName() + " has no SchedulerAppConn.");
253+
// }
254+
if (appConn != null) {
255+
AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0);
256+
return new ImmutablePair<>(appConn.getOrCreateStructureStandard().getOrchestrationService(appInstance), appInstance);
257+
} else {
258+
return new ImmutablePair<>(null, null);
249259
}
250-
AppInstance appInstance = appConn.getAppDesc().getAppInstances().get(0);
251-
return new ImmutablePair<>(appConn.getOrCreateStructureStandard().getOrchestrationService(appInstance), appInstance);
260+
252261
}
253262

254263
/**

‎dss-framework/dss-framework-orchestrator-server/src/main/java/com/webank/wedatasphere/dss/orchestrator/server/service/impl/OrchestratorServiceImpl.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616

1717
package com.webank.wedatasphere.dss.orchestrator.server.service.impl;
1818

19+
import com.webank.wedatasphere.dss.appconn.core.AppConn;
1920
import com.webank.wedatasphere.dss.common.constant.project.ProjectUserPrivEnum;
2021
import com.webank.wedatasphere.dss.common.exception.DSSErrorException;
2122
import com.webank.wedatasphere.dss.common.label.DSSLabel;
@@ -51,6 +52,7 @@
5152
import com.webank.wedatasphere.dss.standard.app.development.service.RefQueryService;
5253
import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard;
5354
import com.webank.wedatasphere.dss.standard.app.sso.Workspace;
55+
import com.webank.wedatasphere.dss.standard.common.desc.AppDesc;
5456
import com.webank.wedatasphere.dss.standard.common.desc.AppInstance;
5557
import com.webank.wedatasphere.dss.standard.common.entity.ref.ResponseRef;
5658
import com.webank.wedatasphere.dss.standard.common.exception.operation.ExternalOperationWarnException;
@@ -110,7 +112,9 @@ public OrchestratorVo createOrchestrator(String userName,
110112
Map<String, Object> dssJobContent = MapUtils.newCommonMapBuilder()
111113
.put(OrchestratorRefConstant.DSS_ORCHESTRATOR_INFO_KEY, dssOrchestratorInfo)
112114
.put(OrchestratorRefConstant.ORCHESTRATOR_VERSION_KEY, version)
113-
.put(OrchestratorRefConstant.ORCHESTRATION_SCHEDULER_APP_CONN, dssOrchestrator.getSchedulerAppConn().getAppDesc().getAppName()).build();
115+
.put(OrchestratorRefConstant.ORCHESTRATION_SCHEDULER_APP_CONN, Optional.ofNullable(dssOrchestrator)
116+
.map(DSSOrchestrator::getSchedulerAppConn).map(AppConn::getAppDesc).map(AppDesc::getAppName)
117+
.map(Object::toString).orElse("NULL")).build();
114118
DSSJobContentRequestRef requestRef = (DSSJobContentRequestRef) developmentRequestRef;
115119
requestRef.setDSSJobContent(dssJobContent);
116120
return ((RefCreationOperation) developmentOperation).createRef(requestRef);

‎dss-framework/dss-framework-workspace-server/src/main/java/com/webank/wedatasphere/dss/framework/workspace/dao/DSSWorkspaceUserMapper.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ void setUserRoleInWorkspace(@Param("workspaceId") int workspaceId, @Param("roleI
5959
"<script>",
6060
"select distinct created_by as creator, username as username, create_time as joinTime,workspace_id as workspaceId " +
6161
"from dss_workspace_user_role where workspace_id = #{workspaceId} ",
62-
"<if test='username != null'>and username=#{username}</if> order by id desc",
62+
"<if test='username != null'>and username=#{username}</if> ",
6363
"</script>"
6464
})
6565
List<DSSWorkspaceUser> getWorkspaceUsers(@Param("workspaceId") String workspaceId,@Param("username") String username);

‎dss-orchestrator/dss-orchestrator-loader/src/main/java/com/webank/wedatasphere/dss/orchestrator/loader/DefaultOrchestratorLoader.java‎

Lines changed: 12 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -63,17 +63,19 @@ protected DSSOrchestratorContext createOrchestratorContext() {
6363
appConnList.stream().filter(relation.isLinkedAppConn()).forEach(dssOrchestrator::addLinkedAppConn);
6464
AppConn appConn = AppConnManager.getAppConnManager().getAppConn(relation.getBindingAppConnName());
6565
dssOrchestrator.setAppConn(appConn);
66-
List<SchedulerAppConn> schedulerAppConns = AppConnManager.getAppConnManager().listAppConns(SchedulerAppConn.class);
67-
SchedulerAppConn schedulerAppConn;
68-
if(StringUtils.isBlank(relation.getBindingSchedulerAppConnName())) {
69-
schedulerAppConn = schedulerAppConns.get(0);
70-
} else {
71-
schedulerAppConn = schedulerAppConns.stream().filter(appConn1 -> appConn1.getAppDesc().getAppName().equals(relation.getBindingSchedulerAppConnName()))
72-
.findAny().orElseThrow(() -> new ExternalOperationFailedException(50032, "cannot find the matched SchedulerAppConn with name " + relation.getBindingSchedulerAppConnName()));
66+
if (appConnList.stream().anyMatch(t -> t instanceof SchedulerAppConn)) {
67+
List<SchedulerAppConn> schedulerAppConns = AppConnManager.getAppConnManager().listAppConns(SchedulerAppConn.class);
68+
SchedulerAppConn schedulerAppConn;
69+
if(StringUtils.isBlank(relation.getBindingSchedulerAppConnName())) {
70+
schedulerAppConn = schedulerAppConns.get(0);
71+
} else {
72+
schedulerAppConn = schedulerAppConns.stream().filter(appConn1 -> appConn1.getAppDesc().getAppName().equals(relation.getBindingSchedulerAppConnName()))
73+
.findAny().orElseThrow(() -> new ExternalOperationFailedException(50032, "cannot find the matched SchedulerAppConn with name " + relation.getBindingSchedulerAppConnName()));
74+
}
75+
dssOrchestrator.setSchedulerAppConn(schedulerAppConn);
76+
LOGGER.info("Load dss orchestrator: {}, with the binding AppConn: {} and binding SchedulerAppConn {}.",
77+
typeName, relation.getBindingAppConnName(), schedulerAppConn.getAppDesc().getAppName());
7378
}
74-
dssOrchestrator.setSchedulerAppConn(schedulerAppConn);
75-
LOGGER.info("Load dss orchestrator: {}, with the binding AppConn: {} and binding SchedulerAppConn {}.",
76-
typeName, relation.getBindingAppConnName(), schedulerAppConn.getAppDesc().getAppName());
7779
dssLabels.forEach(dssOrchestrator::addLinkedDssLabels);
7880
return dssOrchestrator;
7981
}

‎plugins/dolphinscheduler/dolphinscheduler-prod-metrics/pom.xml‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,34 @@
3333
<groupId>org.apache.dolphinscheduler</groupId>
3434
<artifactId>dolphinscheduler-api</artifactId>
3535
<version>${dolphinscheduler.version}</version>
36+
<scope>provided</scope>
37+
</dependency>
38+
<dependency>
39+
<groupId>com.webank.wedatasphere.dss</groupId>
40+
<artifactId>spring-origin-sso-integration-plugin</artifactId>
41+
<version>${dss.version}</version>
42+
<exclusions>
43+
<exclusion>
44+
<groupId>mysql</groupId>
45+
<artifactId>mysql-connector-java</artifactId>
46+
</exclusion>
47+
<exclusion>
48+
<groupId>org.springframework</groupId>
49+
<artifactId>spring-core</artifactId>
50+
</exclusion>
51+
<exclusion>
52+
<artifactId>spring-webmvc</artifactId>
53+
<groupId>org.springframework</groupId>
54+
</exclusion>
55+
<exclusion>
56+
<artifactId>spring-context-support</artifactId>
57+
<groupId>org.springframework</groupId>
58+
</exclusion>
59+
<exclusion>
60+
<artifactId>spring-aop</artifactId>
61+
<groupId>org.springframework</groupId>
62+
</exclusion>
63+
</exclusions>
3664
</dependency>
3765
</dependencies>
3866

@@ -51,6 +79,25 @@
5179
<groupId>org.apache.maven.plugins</groupId>
5280
<artifactId>maven-jar-plugin</artifactId>
5381
</plugin>
82+
<plugin>
83+
<groupId>org.apache.maven.plugins</groupId>
84+
<artifactId>maven-assembly-plugin</artifactId>
85+
<version>2.3</version>
86+
<configuration>
87+
<descriptorRefs>
88+
<descriptorRef>jar-with-dependencies</descriptorRef>
89+
</descriptorRefs>
90+
</configuration>
91+
<executions>
92+
<execution>
93+
<id>assembly</id>
94+
<phase>package</phase>
95+
<goals>
96+
<goal>single</goal>
97+
</goals>
98+
</execution>
99+
</executions>
100+
</plugin>
54101
</plugins>
55102
<resources>
56103
<resource>

0 commit comments

Comments
 (0)