Skip to content

Commit a587a49

Browse files
zqburdezqburde
committed
Adopt to Linkis 1.4.0.
1 parent 5b59f5f commit a587a49

File tree

7 files changed

+20
-179
lines changed

7 files changed

+20
-179
lines changed

‎dss-appconn/appconns/dss-sendemail-appconn/sendemail-appconn-core/src/main/scala/com/webank/wedatasphere/dss/appconn/sendemail/emailcontent/parser/AbstractEmailContentParser.scala‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ import org.apache.linkis.common.io.resultset.ResultSetReader
2626
import org.apache.linkis.common.io.{MetaData, Record}
2727
import org.apache.linkis.common.utils.Utils
2828
import org.apache.linkis.storage.LineRecord
29-
import org.apache.linkis.storage.resultset.ResultSetReader
29+
import org.apache.linkis.storage.resultset.ResultSetReaderFactory
3030
import org.apache.commons.io.IOUtils
3131

3232
abstract class AbstractEmailContentParser[T] extends EmailContentParser {
@@ -41,8 +41,8 @@ abstract class AbstractEmailContentParser[T] extends EmailContentParser {
4141
case _ =>
4242
}
4343

44-
protected def getResultSetReader(fsPathStore: FsPathStoreEmailContent): ResultSetReader[_ <: MetaData, _ <: Record] = {
45-
val reader = ResultSetReader.getResultSetReader(fsPathStore.getFsPath.getSchemaPath)
44+
protected def getResultSetReader(fsPathStore: FsPathStoreEmailContent): ResultSetReader[_, _ ] = {
45+
val reader = ResultSetReaderFactory.getResultSetReader(fsPathStore.getFsPath.getSchemaPath)
4646
reader.getMetaData
4747
reader
4848
}

‎dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/AppConnEngineConnPlugin.scala‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,7 +45,7 @@ class AppConnEngineConnPlugin extends EngineConnPlugin {
4545
private val defaultLabels: util.List[Label[_]] = new util.ArrayList[Label[_]]()
4646

4747

48-
override def init(params: util.Map[String, Any]): Unit = {
48+
override def init(params: util.Map[String, AnyRef]): Unit = {
4949
val engineTypeLabel = EngineTypeLabelCreator.createEngineTypeLabel(EngineType.APPCONN.toString)
5050
this.defaultLabels.add(engineTypeLabel)
5151
}

‎dss-appconn/linkis-appconn-engineplugin/src/main/scala/org/apache/linkis/manager/engineplugin/appconn/executor/AbstractExecutionRequestRefContext.scala‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
package org.apache.linkis.manager.engineplugin.appconn.executor
1818

1919
import java.util
20-
2120
import com.webank.wedatasphere.dss.standard.app.development.listener.core.ExecutionRequestRefContext
2221
import com.webank.wedatasphere.dss.standard.app.development.listener.exception.AppConnExecutionErrorException
2322
import org.apache.linkis.common.io.resultset.{ResultSet, ResultSetReader, ResultSetWriter}
@@ -28,7 +27,7 @@ import org.apache.linkis.manager.engineplugin.appconn.conf.AppConnEngineConnConf
2827
import org.apache.linkis.rpc.Sender
2928
import org.apache.linkis.storage.FSFactory
3029
import org.apache.linkis.storage.fs.FileSystem
31-
import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReader}
30+
import org.apache.linkis.storage.resultset.{ResultSetFactory, ResultSetReaderFactory}
3231

3332
abstract class AbstractExecutionRequestRefContext(engineExecutorContext: EngineExecutionContext,
3433
user: String,
@@ -70,8 +69,10 @@ abstract class AbstractExecutionRequestRefContext(engineExecutorContext: EngineE
7069
resultSetAlias: String): ResultSetWriter[M, R] =
7170
engineExecutorContext.createResultSetWriter(resultSet, resultSetAlias).asInstanceOf[ResultSetWriter[M, R]]
7271

73-
override def getResultSetReader[M <: MetaData, R <: Record](fsPath: FsPath): ResultSetReader[M, R] =
74-
ResultSetReader.getResultSetReader(fsPath.getSchemaPath).asInstanceOf[ResultSetReader[M, R]]
72+
override def getResultSetReader[M <: MetaData, R <: Record](fsPath: FsPath): ResultSetReader[M, R] = {
73+
74+
ResultSetReaderFactory.getResultSetReader(fsPath.getSchemaPath).asInstanceOf[ResultSetReader[M, R]]
75+
}
7576

7677
private def createResultSetWriter[M <: MetaData, R <: Record](resultSetType: String, resultSetAlias: String): ResultSetWriter[M, R] =
7778
engineExecutorContext.createResultSetWriter(resultSetType, resultSetAlias).asInstanceOf[ResultSetWriter[M, R]]

‎dss-apps/dss-apiservice-server/pom.xml‎

Lines changed: 1 addition & 68 deletions
Original file line numberDiff line numberDiff line change
@@ -204,23 +204,13 @@
204204
</exclusion>
205205
</exclusions>
206206
</dependency>
207-
<!--Metadata common-->
208-
<!-- <dependency>-->
209-
<!-- <groupId>org.apache.linkis</groupId>-->
210-
<!-- <artifactId>linkis-metadatamanager-common</artifactId>-->
211-
<!-- <version>${linkis.version}</version>-->
212-
<!-- </dependency>-->
207+
213208
<dependency>
214209
<artifactId>hk2-api</artifactId>
215210
<groupId>org.glassfish.hk2</groupId>
216211
<version>2.4.0-b34</version>
217212
</dependency>
218213

219-
<!-- <dependency>-->
220-
<!-- <groupId>org.apache.linkis</groupId>-->
221-
<!-- <artifactId>linkis-datasourcemanager-common</artifactId>-->
222-
<!-- <version>${linkis.version}</version>-->
223-
<!-- </dependency>-->
224214
<dependency>
225215
<groupId>org.modelmapper</groupId>
226216
<artifactId>modelmapper</artifactId>
@@ -246,7 +236,6 @@
246236
<scope>test</scope>
247237
</dependency>
248238

249-
250239
<dependency>
251240
<groupId>com.h2database</groupId>
252241
<artifactId>h2</artifactId>
@@ -285,62 +274,6 @@
285274
<version>1.5.2</version>
286275
<scope>test</scope>
287276
</dependency>
288-
<!-- <dependency>-->
289-
<!-- <groupId>org.springframework.cloud</groupId>-->
290-
<!-- <artifactId>spring-cloud-starter-netflix-eureka-client</artifactId>-->
291-
<!-- <version>${spring.cloud.version}</version>-->
292-
<!-- <exclusions>-->
293-
<!-- <exclusion>-->
294-
<!-- <artifactId>logback-classic</artifactId>-->
295-
<!-- <groupId>ch.qos.logback</groupId>-->
296-
<!-- </exclusion>-->
297-
<!-- <exclusion>-->
298-
<!-- <artifactId>log4j-to-slf4j</artifactId>-->
299-
<!-- <groupId>org.apache.logging.log4j</groupId>-->
300-
<!-- </exclusion>-->
301-
<!-- <exclusion>-->
302-
<!-- <artifactId>gson</artifactId>-->
303-
<!-- <groupId>com.google.code.gson</groupId>-->
304-
<!-- </exclusion>-->
305-
<!-- <exclusion>-->
306-
<!-- <artifactId>jsr311-api</artifactId>-->
307-
<!-- <groupId>javax.ws.rs</groupId>-->
308-
<!-- </exclusion>-->
309-
<!-- <exclusion>-->
310-
<!-- <artifactId>xstream</artifactId>-->
311-
<!-- <groupId>com.thoughtworks.xstream</groupId>-->
312-
<!-- </exclusion>-->
313-
<!-- <exclusion>-->
314-
<!-- <artifactId>commons-math</artifactId>-->
315-
<!-- <groupId>org.apache.commons</groupId>-->
316-
<!-- </exclusion>-->
317-
<!-- <exclusion>-->
318-
<!-- <artifactId>jackson-core</artifactId>-->
319-
<!-- <groupId>com.fasterxml.jackson.core</groupId>-->
320-
<!-- </exclusion>-->
321-
<!-- <exclusion>-->
322-
<!-- <artifactId>spring-boot-autoconfigure</artifactId>-->
323-
<!-- <groupId>org.springframework.boot</groupId>-->
324-
<!-- </exclusion>-->
325-
<!-- <exclusion>-->
326-
<!-- <artifactId>spring-boot-starter-aop</artifactId>-->
327-
<!-- <groupId>org.springframework.boot</groupId>-->
328-
<!-- </exclusion>-->
329-
<!-- <exclusion>-->
330-
<!-- <artifactId>spring-boot-starter</artifactId>-->
331-
<!-- <groupId>org.springframework.boot</groupId>-->
332-
<!-- </exclusion>-->
333-
<!-- <exclusion>-->
334-
<!-- <artifactId>spring-boot-starter-cache</artifactId>-->
335-
<!-- <groupId>org.springframework.boot</groupId>-->
336-
<!-- </exclusion>-->
337-
<!-- <exclusion>-->
338-
<!-- <groupId>org.springframework.cloud</groupId>-->
339-
<!-- <artifactId>spring-cloud-starter-loadbalancer</artifactId>-->
340-
<!-- </exclusion>-->
341-
<!-- </exclusions>-->
342-
<!-- </dependency>-->
343-
344277
<dependency>
345278
<artifactId>xstream</artifactId>
346279
<groupId>com.thoughtworks.xstream</groupId>

‎dss-apps/dss-apiservice-server/src/main/java/com/webank/wedatasphere/dss/apiservice/core/service/impl/ApiServiceQueryServiceImpl.java‎

Lines changed: 4 additions & 99 deletions
Original file line numberDiff line numberDiff line change
@@ -23,10 +23,8 @@
2323
import com.webank.wedatasphere.dss.apiservice.core.bo.LinkisExecuteResult;
2424
import com.webank.wedatasphere.dss.apiservice.core.config.ApiServiceConfiguration;
2525
import com.webank.wedatasphere.dss.apiservice.core.constant.ParamType;
26-
import com.webank.wedatasphere.dss.apiservice.core.constant.ParamTypeEnum;
2726
import com.webank.wedatasphere.dss.apiservice.core.constant.RequireEnum;
2827
import com.webank.wedatasphere.dss.apiservice.core.dao.*;
29-
import com.webank.wedatasphere.dss.apiservice.core.exception.ApiExecuteException;
3028
import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceQueryException;
3129
import com.webank.wedatasphere.dss.apiservice.core.execute.ApiServiceExecuteJob;
3230
import com.webank.wedatasphere.dss.apiservice.core.execute.DefaultApiServiceJob;
@@ -37,22 +35,18 @@
3735
import com.webank.wedatasphere.dss.apiservice.core.util.DateUtil;
3836
import com.webank.wedatasphere.dss.apiservice.core.util.SQLCheckUtil;
3937
import com.webank.wedatasphere.dss.apiservice.core.vo.*;
40-
//import com.webank.wedatasphere.dss.oneservice.core.jdbc.JdbcUtil;
4138
import com.webank.wedatasphere.dss.apiservice.core.exception.ApiServiceRuntimeException;
4239
import com.webank.wedatasphere.dss.apiservice.core.service.ApiServiceQueryService;
4340
import com.webank.wedatasphere.dss.apiservice.core.util.AssertUtil;
4441
import com.webank.wedatasphere.dss.apiservice.core.util.ModelMapperUtil;
45-
//import com.webank.wedatasphere.dss.oneservice.core.vo.*;
4642
import com.webank.wedatasphere.dss.apiservice.core.vo.ApiServiceVo;
4743
import org.apache.linkis.bml.client.BmlClient;
4844
import org.apache.linkis.bml.client.BmlClientFactory;
4945
import org.apache.linkis.bml.protocol.BmlDownloadResponse;
5046
import org.apache.linkis.common.io.FsPath;
5147
import org.apache.linkis.storage.source.FileSource;
52-
import org.apache.linkis.storage.source.FileSource$;
5348
import org.apache.linkis.ujes.client.UJESClient;
5449
import org.apache.linkis.ujes.client.response.JobExecuteResult;
55-
import org.apache.commons.collections4.MapUtils;
5650
import org.apache.commons.lang.StringUtils;
5751
import org.apache.commons.math3.util.Pair;
5852
import org.slf4j.Logger;
@@ -141,11 +135,9 @@ public class ApiServiceQueryServiceImpl implements ApiServiceQueryService {
141135
@Autowired
142136
private ApiServiceTokenManagerDao apiServiceTokenManagerDao;
143137

144-
145138
@Autowired
146139
private ApiService apiService;
147140

148-
149141
@Autowired
150142
private ApiServiceAccessDao apiServiceAccessDao;
151143

@@ -169,9 +161,7 @@ public LinkisExecuteResult query(String path,
169161
ApiServiceToken tokenDetail,
170162
String loginUser) {
171163
// 根据path查询resourceId和version
172-
173164
// 得到metadata
174-
175165
// 执行查询
176166
//path 必须唯一
177167
ApiServiceVo apiServiceVo = apiServiceDao.queryByPath(path);
@@ -241,10 +231,6 @@ public LinkisExecuteResult query(String path,
241231
}
242232
});
243233

244-
245-
// AssertUtil.isTrue(MapUtils.isNotEmpty((Map) collect.getKey()), "数据源不能为空");
246-
247-
248234
ApiServiceExecuteJob job = new DefaultApiServiceJob();
249235
//sql代码封装成scala执行
250236
job.setCode(ExecuteCodeHelper.packageCodeToExecute(executeCode, maxApiVersionVo.getMetadataInfo()));
@@ -270,7 +256,6 @@ public LinkisExecuteResult query(String path,
270256
apiAccessVo.setAccessTime(DateUtil.getNow());
271257
apiServiceAccessDao.addAccessRecord(apiAccessVo);
272258

273-
274259
JobExecuteResult jobExecuteResult = LinkisJobSubmit.execute(job,ujesClient);
275260

276261
//记录执行任务用户和代理用户关系，没有代理用户的统一设置为登录用户
@@ -280,17 +265,13 @@ public LinkisExecuteResult query(String path,
280265
apiServiceJob.setJobExecuteResult(jobExecuteResult);
281266
runJobs.put(jobExecuteResult.getTaskID(),apiServiceJob);
282267

283-
284268
LinkisExecuteResult linkisExecuteResult = new LinkisExecuteResult(jobExecuteResult.getTaskID(), jobExecuteResult.getExecID());
285269
return linkisExecuteResult;
286270
} catch (IOException e) {
287271
throw new ApiServiceRuntimeException(e.getMessage(), e);
288272
}
289273
}
290274

291-
292-
293-
294275
@Override
295276
public ApiServiceVo queryByVersionId(String userName,Long versionId) throws ApiServiceQueryException {
296277
ApiVersionVo apiVersionVo = apiServiceVersionDao.queryApiVersionByVersionId(versionId);
@@ -325,10 +306,8 @@ public List<QueryParamVo> queryParamList(String scriptPath, Long versionId) {
325306

326307
AssertUtil.notNull(targetApiVersionVo, "目标参数版本不存在，path=" + scriptPath+",version:"+versionId);
327308

328-
// todo~！
329309
List<ParamVo> paramVoList = apiServiceParamDao.queryByVersionId(targetApiVersionVo.getId());
330310

331-
332311
List<QueryParamVo> queryParamVoList = new ArrayList<>();
333312

334313
Map<String, ParamVo> paramMap = paramVoList.stream()
@@ -390,16 +369,16 @@ private Pair<Object, ArrayList<String[]>> queryBml(String userName, String resou
390369

391370
InputStream inputStream = resource.inputStream();
392371

393-
try (FileSource fileSource = FileSource$.MODULE$.create(new FsPath(scriptPath), inputStream)) {
372+
try (FileSource fileSource = FileSource.create(new FsPath(scriptPath), inputStream)) {
394373
//todo 数组取了第一个
395-
collect = fileSource.collect()[0];
374+
Pair<Object, List<String[]>> sourcePair = fileSource.collect()[0];
375+
collect = new Pair<>(sourcePair.getKey(), new ArrayList<>(sourcePair.getValue()));
396376
bmlCache.put(key, collect);
397377
}
398378
}
399379
}
400380
}
401381

402-
403382
return collect;
404383
}
405384

@@ -425,60 +404,6 @@ private Map<String, String> queryConfigParam(long apiId, String version) {
425404
return collect;
426405
}
427406

428-
429-
430-
431-
// private Tuple3 getDatasourceInfo(final Map<String, Object> datasourceMap) {
432-
// Tuple3 tuple3 = datasourceCache.getIfPresent(datasourceMap);
433-
//
434-
// if (tuple3 == null) {
435-
// synchronized (this) {
436-
// tuple3 = datasourceCache.getIfPresent(datasourceMap);
437-
// if (tuple3 == null) {
438-
// tuple3 = JdbcUtil.getDatasourceInfo(datasourceMap);
439-
// datasourceCache.put(datasourceMap, tuple3);
440-
// }
441-
// }
442-
// }
443-
//
444-
// return tuple3;
445-
// }
446-
447-
// private List<Map<String, Object>> executeJob(String executeCode,
448-
// Object datasourceMap, Map<String, Object> params) {
449-
//
450-
//// Tuple3 tuple3 = getDatasourceInfo((Map<String, Object>) datasourceMap);
451-
//// final String jdbcUrl = tuple3._1().toString();
452-
//// final String username = tuple3._2().toString();
453-
//// final String password = tuple3._3().toString();
454-
//
455-
//// NamedParameterJdbcTemplate namedParameterJdbcTemplate = datasourceService.getNamedParameterJdbcTemplate(jdbcUrl, username, password);
456-
//
457-
// String namedSql = genNamedSql(executeCode, params);
458-
//
459-
//// return namedParameterJdbcTemplate.query(namedSql, new MapSqlParameterSource(params), new ColumnAliasMapRowMapper());
460-
//
461-
// }
462-
463-
private static String genNamedSql(String executeCode, Map<String, Object> params) {
464-
// 没有参数，无需生成namedSql
465-
if (MapUtils.isEmpty(params)) {
466-
return executeCode;
467-
}
468-
469-
for (String paramName : params.keySet()) {
470-
for (String $name : new String[]{"'${" + paramName + "}'", "${" + paramName + "}", "\"${" + paramName + "}\""}) {
471-
if (executeCode.contains($name)) {
472-
executeCode = StringUtils.replace(executeCode, $name, ":" + paramName);
473-
break;
474-
}
475-
}
476-
}
477-
478-
return executeCode;
479-
}
480-
481-
482407
public static class ColumnAliasMapRowMapper implements RowMapper<Map<String, Object>> {
483408
@Override
484409
public Map<String, Object> mapRow(ResultSet rs, int rowNum) throws SQLException {
@@ -520,27 +445,7 @@ protected Object getColumnValue(ResultSet rs, int index) throws SQLException {
520445

521446
@Override
522447
public ApiServiceJob getJobByTaskId(String taskId){
523-
ApiServiceJob apiServiceJob=runJobs.get(taskId);
524-
return apiServiceJob;
448+
return runJobs.get(taskId);
525449
}
526450

527-
528-
private static String getRunTypeFromScriptsPath(String scriptsPath) {
529-
530-
String res = "sql";
531-
String fileFlag = scriptsPath.substring(scriptsPath.lastIndexOf(".") + 1);
532-
switch (fileFlag) {
533-
case "sh":
534-
res = "shell";
535-
break;
536-
case "py":
537-
res= "pyspark";
538-
break;
539-
default:
540-
res = fileFlag;
541-
break;
542-
}
543-
return res;
544-
545-
}
546451
}

0 commit comments

Comments
 (0)