All interfaces of DSS can be changed into spring MVC #445

Version 1.0.1 of DSS is adapted to Apache of linkis #444

Author: JackChen0810

assembly/config/config.sh

@@ -76,6 +76,6 @@ EMAIL_PASSWORD=xxxxx
 EMAIL_PROTOCOL=smtp
 ############## ############## dss_appconn_instance configuration   end   ############## ##############
 
-DSS_VERSION=1.0.0
+DSS_VERSION=1.0.1
 
 DSS_FILE_NAME="dss-$DSS_VERSION"

assembly/dss-package/pom.xml

@@ -21,13 +21,13 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>1.0.0</version>
+        <version>1.0.1</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dss-package</artifactId>
     <dependencies>
         <dependency>
-            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <groupId>org.apache.linkis</groupId>
             <artifactId>linkis-rpc</artifactId>
             <version>${linkis.version}</version>
             <exclusions>
@@ -88,7 +88,7 @@
         </dependency>
 
         <dependency>
-            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <groupId>org.apache.linkis</groupId>
             <artifactId>linkis-hadoop-common</artifactId>
             <version>${linkis.version}</version>
         </dependency>
@@ -111,12 +111,12 @@
         </dependency>
 
         <dependency>
-            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <groupId>org.apache.linkis</groupId>
             <artifactId>linkis-mybatis</artifactId>
             <version>${linkis.version}</version>
         </dependency>
         <dependency>
-            <groupId>com.webank.wedatasphere.linkis</groupId>
+            <groupId>org.apache.linkis</groupId>
             <artifactId>linkis-storage</artifactId>
             <version>${linkis.version}</version>
         </dependency>

assembly/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>1.0.0</version>
+        <version>1.0.1</version>
     </parent>
     <packaging>pom</packaging>
     <modelVersion>4.0.0</modelVersion>

conf/dss-flow-execution-server.properties

@@ -20,16 +20,16 @@ spring.server.port=9205
 spring.spring.application.name=dss-flow-entrance
 
 ##mybatis
-wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/*.xml,classpath*:com/webank/wedatasphere/linkis/jobhistory/dao/impl/*.xml
+wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/flow/execution/entrance/dao/impl/*.xml,classpath*:org/apache/linkis/jobhistory/dao/impl/*.xml
 
 wds.linkis.server.mybatis.typeAliasesPackage=
 
-wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.flow.execution.entrance.dao,com.webank.wedatasphere.linkis.jobhistory.dao
+wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.flow.execution.entrance.dao,org.apache.linkis.jobhistory.dao
 
 
-wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.linkis.entrance.restful,com.webank.wedatasphere.dss.flow.execution.entrance.restful
+wds.linkis.server.restful.scan.packages=org.apache.linkis.entrance.restful,com.webank.wedatasphere.dss.flow.execution.entrance.restful
 
-#wds.linkis.server.component.exclude.classes=com.webank.wedatasphere.linkis.DataWorkCloudApplication
+#wds.linkis.server.component.exclude.classes=org.apache.linkis.DataWorkCloudApplication
 
 wds.linkis.engine.application.name=flowExecutionEngine
 wds.linkis.enginemanager.application.name=flowExecution
@@ -45,9 +45,9 @@ wds.linkis.server.socket.mode=true
 wds.linkis.client.flow.adminuser=ws
 wds.linkis.client.flow.author.user.token=172.0.0.1
 
-wds.linkis.server.component.exclude.classes=com.webank.wedatasphere.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,com.webank.wedatasphere.linkis.entranceclient.conf.ClientSpringConfiguration
+wds.linkis.server.component.exclude.classes=org.apache.linkis.entranceclient.conf.ClientForEntranceSpringConfiguration,org.apache.linkis.entranceclient.conf.ClientSpringConfiguration
 
-wds.linkis.server.component.exclude.packages=com.webank.wedatasphere.linkis.entrance.restful.
+wds.linkis.server.component.exclude.packages=org.apache.linkis.entrance.restful.
 spring.spring.main.allow-bean-definition-overriding=true
 
 wds.linkis.entrance.config.log.path=file:///appcom/tmp/dss/

conf/dss.properties

@@ -27,4 +27,10 @@ wds.linkis.server.mybatis.datasource.username=
 wds.dss.esb.appid=
 wds.dss.esb.token=
 
-wds.dss.appconn.scheduler.job.label=dev
+wds.dss.appconn.scheduler.job.label=dev
+
+
+wds.linkis.reflect.scan.package=org.apache.linkis,com.webank.wedatasphere.dss
+spring.spring.mvc.servlet.path=/api/rest_j/v1
+spring.spring.servlet.multipart.max-file-size=200MB
+spring.spring.servlet.multipart.max-request-size=200MB

dss-appconn/appconns/dss-datachecker-appconn/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>1.0.0</version>
+        <version>1.0.1</version>
         <relativePath>../../../pom.xml</relativePath>
     </parent>
     <modelVersion>4.0.0</modelVersion>
@@ -38,7 +38,7 @@
             <exclusions>
                 <exclusion>
                     <artifactId>linkis-common</artifactId>
-                    <groupId>com.webank.wedatasphere.linkis</groupId>
+                    <groupId>org.apache.linkis</groupId>
                 </exclusion>
                 <exclusion>
                     <artifactId>json4s-jackson_2.11</artifactId>

dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataChecker.java

@@ -0,0 +1,93 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.webank.wedatasphere.dss.appconn.datachecker;
+
+
+import com.webank.wedatasphere.dss.appconn.datachecker.connector.DataCheckerDao;
+import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction;
+import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState;
+import org.apache.log4j.Logger;
+
+import java.util.Properties;
+
+public class DataChecker {
+    public final static String SOURCE_TYPE = "source.type";
+    public final static String DATA_OBJECT = "check.object";
+    public final static String WAIT_TIME = "max.check.hours";
+    public final static String QUERY_FREQUENCY = "query.frequency";
+    public final static String TIME_SCAPE = "time.scape";
+    public final static String MASK_URL = "bdp.mask.url";
+    public final static String MASK_APP_ID = "bdp.mask.app.id";
+    public final static String MASK_APP_TOKEN = "bdp.mask.app.token";
+
+    private Properties p;
+    private static final Logger logger = Logger.getRootLogger();
+    DataCheckerDao wbDao = DataCheckerDao.getInstance();
+    DataCheckerExecutionAction dataCheckerAction = null;
+    public long maxWaitTime;
+    public int queryFrequency;
+
+    public DataChecker(Properties p, DataCheckerExecutionAction action) {
+        this.p = p;
+        dataCheckerAction = action;
+        maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000;
+        //test over time
+//        maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 120 * 1000;
+        queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000"));
+
+    }
+
+    public void run() {
+        dataCheckerAction.setState(RefExecutionState.Running);
+        try {
+            if(p == null) {
+                throw new RuntimeException("Properties is null. Can't continue");
+            }
+            if (!p.containsKey(SOURCE_TYPE)) {
+                logger.info("Properties "  + SOURCE_TYPE + " value is Null !");
+            }
+            if (!p.containsKey(DATA_OBJECT)) {
+                logger.info("Properties " + DATA_OBJECT + " value is Null !");
+            }
+            begineCheck(dataCheckerAction);
+        }catch (Exception ex){
+            dataCheckerAction.setState(RefExecutionState.Failed);
+            throw new  RuntimeException("get DataChecker result failed", ex);
+        }
+
+    }
+
+    public void begineCheck(RefExecutionAction action){
+        boolean success=false;
+        try {
+            success= wbDao.validateTableStatusFunction(p, logger,action);
+        }catch (Exception ex){
+            dataCheckerAction.setState(RefExecutionState.Failed);
+            logger.error("datacheck error",ex);
+            throw new  RuntimeException("get DataChecker result failed", ex);
+        }
+        if(success) {
+            dataCheckerAction.setState(RefExecutionState.Success);
+        }else {
+            dataCheckerAction.setState(RefExecutionState.Running);
+        }
+    }
+
+    public void cancel() {
+    }
+
+}

dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerAppConn.java

@@ -0,0 +1,37 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.webank.wedatasphere.dss.appconn.datachecker;
+
+import com.webank.wedatasphere.dss.appconn.datachecker.standard.DataCheckerDevelopmentStandard;
+import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn;
+import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn;
+import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard;
+
+public class DataCheckerAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn {
+
+    private DataCheckerDevelopmentStandard standard;
+
+    @Override
+    protected void initialize() {
+        standard = new DataCheckerDevelopmentStandard();
+    }
+
+    @Override
+    public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() {
+        return standard;
+    }
+}

dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/DataCheckerCompletedExecutionResponseRef.java

@@ -0,0 +1,50 @@
+/*
+ * Copyright 2019 WeBank
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+package com.webank.wedatasphere.dss.appconn.datachecker;
+
+import com.webank.wedatasphere.dss.standard.app.development.listener.common.CompletedExecutionResponseRef;
+
+import java.util.Map;
+
+public class DataCheckerCompletedExecutionResponseRef extends CompletedExecutionResponseRef {
+
+    private Exception exception;
+    public void setException(Exception exception) {
+        this.exception = exception;
+    }
+
+
+
+    public DataCheckerCompletedExecutionResponseRef(int status) {
+        super(status);
+    }
+
+    public DataCheckerCompletedExecutionResponseRef(String responseBody, int status) {
+        super(responseBody, status);
+    }
+
+    public void setStatus(int status){
+        this.status = status;
+    }
+
+
+    @Override
+    public Map<String, Object> toMap() {
+        return null;
+    }
+
+}

…inkis/common/conf/DSSConfiguration.scala …r/common/MaskCheckNotExistException.javadss-commons/dss-common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/DSSConfiguration.scala renamed to dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/MaskCheckNotExistException.java dss-commons/dss-common/src/main/scala/com/webank/wedatasphere/linkis/common/conf/DSSConfiguration.scala renamed to dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatasphere/dss/appconn/datachecker/common/MaskCheckNotExistException.java

@@ -14,16 +14,11 @@
  *
  */
 
-package com.webank.wedatasphere.linkis.common.conf
+package com.webank.wedatasphere.dss.appconn.datachecker.common;
 
-import java.util.Properties
-
-/**
- * @author allenlliu
- * @date 2021/3/10 17:32
- */
-object DSSConfiguration {
-
-  def getAllProperties: Properties = BDPConfiguration.properties
+public class MaskCheckNotExistException extends Exception {
 
+    public MaskCheckNotExistException(final String message) {
+        super(message);
+    }
 }