Skip to content

Commit 559aa15

Browse files
jinyangraoJinyangRao
jinyangrao
and
JinyangRao
authored
Provide some basic function modules of appconn in dss1.0.0, and realize the implementation of third-party applications connected to DSS (#377)
* Define AppConn core interface and default implementation class. fix #357 * Implement DSS1.0 AppConn class loading and instantiation module. fix #358 * Implement DSS1.0 AppConn class loading and instantiation module. fix #358 * delete useless message. fix #358 * Define the abstract dispatch AppConn. fix #359 * Implement Linkis AppConn plug-in. fix #360 * Delete useless pom location. fix #360 * Realize the application of the three major access specifications of DSS1.0. fix #351 * Delete qualitis appconn module. fix #351 * Delete some useless code. fix #351 Co-authored-by: JinyangRao <[email protected]>
1 parent 033d504 commit 559aa15

File tree

281 files changed

+19295
-0
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

281 files changed

+19295
-0
lines changed

‎dss-appconn/appconns/dss-datachecker-appconn/pom.xml‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
~ Copyright 2019 WeBank
4+
~ Licensed under the Apache License, Version 2.0 (the "License");
5+
~ you may not use this file except in compliance with the License.
6+
~ You may obtain a copy of the License at
7+
~
8+
~ http://www.apache.org/licenses/LICENSE-2.0
9+
~
10+
~ Unless required by applicable law or agreed to in writing, software
11+
~ distributed under the License is distributed on an "AS IS" BASIS,
12+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
~ See the License for the specific language governing permissions and
14+
~ limitations under the License.
15+
~
16+
-->
17+
18+
<project xmlns="http://maven.apache.org/POM/4.0.0"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
21+
<parent>
22+
<artifactId>dss</artifactId>
23+
<groupId>com.webank.wedatasphere.dss</groupId>
24+
<version>1.0.0</version>
25+
<relativePath>../../../pom.xml</relativePath>
26+
</parent>
27+
<modelVersion>4.0.0</modelVersion>
28+
29+
<artifactId>dss-datachecker-appconn</artifactId>
30+
31+
32+
<dependencies>
33+
34+
<dependency>
35+
<groupId>com.webank.wedatasphere.dss</groupId>
36+
<artifactId>dss-appconn-core</artifactId>
37+
<version>${dss.version}</version>
38+
<exclusions>
39+
<exclusion>
40+
<artifactId>linkis-common</artifactId>
41+
<groupId>com.webank.wedatasphere.linkis</groupId>
42+
</exclusion>
43+
<exclusion>
44+
<artifactId>json4s-jackson_2.11</artifactId>
45+
<groupId>org.json4s</groupId>
46+
</exclusion>
47+
</exclusions>
48+
</dependency>
49+
50+
51+
<dependency>
52+
<groupId>com.webank.wedatasphere.dss</groupId>
53+
<artifactId>dss-development-process-standard</artifactId>
54+
<version>${dss.version}</version>
55+
</dependency>
56+
57+
<dependency>
58+
<groupId>com.webank.wedatasphere.dss</groupId>
59+
<artifactId>dss-development-process-standard-execution</artifactId>
60+
<version>${dss.version}</version>
61+
</dependency>
62+
63+
<dependency>
64+
<groupId>log4j</groupId>
65+
<artifactId>log4j</artifactId>
66+
<version>1.2.17</version>
67+
</dependency>
68+
69+
<dependency>
70+
<groupId>com.squareup.okhttp3</groupId>
71+
<artifactId>okhttp</artifactId>
72+
<version>4.2.2</version>
73+
</dependency>
74+
75+
<dependency>
76+
<groupId>org.apache.commons</groupId>
77+
<artifactId>commons-lang3</artifactId>
78+
<version>3.4</version>
79+
</dependency>
80+
81+
<dependency>
82+
<groupId>com.alibaba</groupId>
83+
<artifactId>druid</artifactId>
84+
<version>1.0.28</version>
85+
</dependency>
86+
<dependency>
87+
<groupId>com.webank.wedatasphere.dss</groupId>
88+
<artifactId>dss-common</artifactId>
89+
<version>${dss.version}</version>
90+
<scope>provided</scope>
91+
</dependency>
92+
93+
</dependencies>
94+
95+
<build>
96+
<plugins>
97+
<plugin>
98+
<groupId>org.apache.maven.plugins</groupId>
99+
<artifactId>maven-deploy-plugin</artifactId>
100+
</plugin>
101+
102+
<plugin>
103+
<groupId>net.alchim31.maven</groupId>
104+
<artifactId>scala-maven-plugin</artifactId>
105+
</plugin>
106+
<plugin>
107+
<groupId>org.apache.maven.plugins</groupId>
108+
<artifactId>maven-jar-plugin</artifactId>
109+
</plugin>
110+
<plugin>
111+
<groupId>org.apache.maven.plugins</groupId>
112+
<artifactId>maven-assembly-plugin</artifactId>
113+
<version>2.3</version>
114+
<inherited>false</inherited>
115+
<executions>
116+
<execution>
117+
<id>make-assembly</id>
118+
<phase>package</phase>
119+
<goals>
120+
<goal>single</goal>
121+
</goals>
122+
<configuration>
123+
<descriptors>
124+
<descriptor>src/main/assembly/distribution.xml</descriptor>
125+
</descriptors>
126+
</configuration>
127+
</execution>
128+
</executions>
129+
<configuration>
130+
<skipAssembly>false</skipAssembly>
131+
<finalName>out</finalName>
132+
<appendAssemblyId>false</appendAssemblyId>
133+
<attach>false</attach>
134+
<descriptors>
135+
<descriptor>src/main/assembly/distribution.xml</descriptor>
136+
</descriptors>
137+
</configuration>
138+
</plugin>
139+
</plugins>
140+
<resources>
141+
<resource>
142+
<directory>src/main/java</directory>
143+
<includes>
144+
<include>**/*.xml</include>
145+
</includes>
146+
</resource>
147+
<resource>
148+
<directory>src/main/resources</directory>
149+
<excludes>
150+
<exclude>**/*.properties</exclude>
151+
<exclude>**/application.yml</exclude>
152+
<exclude>**/bootstrap.yml</exclude>
153+
<exclude>**/log4j2.xml</exclude>
154+
</excludes>
155+
</resource>
156+
</resources>
157+
</build>
158+
</project>

‎dss-appconn/appconns/dss-datachecker-appconn/src/main/assembly/distribution.xml‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
<!--
2+
~ Copyright 2019 WeBank
3+
~ Licensed under the Apache License, Version 2.0 (the "License");
4+
~ you may not use this file except in compliance with the License.
5+
~ You may obtain a copy of the License at
6+
~
7+
~ http://www.apache.org/licenses/LICENSE-2.0
8+
~
9+
~ Unless required by applicable law or agreed to in writing, software
10+
~ distributed under the License is distributed on an "AS IS" BASIS,
11+
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
~ See the License for the specific language governing permissions and
13+
~ limitations under the License.
14+
~
15+
-->
16+
17+
<assembly
18+
xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3"
19+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
20+
xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/2.3 http://maven.apache.org/xsd/assembly-1.1.2.xsd">
21+
<id>dss-datachecker-appconn</id>
22+
<formats>
23+
<format>dir</format>
24+
</formats>
25+
<includeBaseDirectory>true</includeBaseDirectory>
26+
<baseDirectory>datachecker</baseDirectory>
27+
28+
<dependencySets>
29+
<dependencySet>
30+
<!-- Enable access to all projects in the current multimodule build! <useAllReactorProjects>true</useAllReactorProjects> -->
31+
<!-- Now, select which projects to include in this module-set. -->
32+
<outputDirectory>lib</outputDirectory>
33+
<useProjectArtifact>true</useProjectArtifact>
34+
<useTransitiveDependencies>true</useTransitiveDependencies>
35+
<unpack>false</unpack>
36+
<useStrictFiltering>true</useStrictFiltering>
37+
<useTransitiveFiltering>true</useTransitiveFiltering>
38+
</dependencySet>
39+
</dependencySets>
40+
41+
<fileSets>
42+
<fileSet>
43+
<directory>${basedir}/src/main/resources</directory>
44+
<includes>
45+
<include>appconn.properties</include>
46+
</includes>
47+
<fileMode>0777</fileMode>
48+
<outputDirectory>/</outputDirectory>
49+
<lineEnding>unix</lineEnding>
50+
</fileSet>
51+
52+
<fileSet>
53+
<directory>${basedir}/src/main/resources</directory>
54+
<includes>
55+
<include>log4j.properties</include>
56+
<include>log4j2.xml</include>
57+
</includes>
58+
<fileMode>0777</fileMode>
59+
<outputDirectory>conf</outputDirectory>
60+
<lineEnding>unix</lineEnding>
61+
</fileSet>
62+
63+
</fileSets>
64+
65+
</assembly>
66+

‎dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatapshere/dss/appconn/datachecker/DataChecker.java‎

Lines changed: 93 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,93 @@
1+
/*
2+
* Copyright 2019 WeBank
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
*/
16+
17+
package com.webank.wedatapshere.dss.appconn.datachecker;
18+
19+
20+
import com.webank.wedatapshere.dss.appconn.datachecker.connector.DataCheckerDao;
21+
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionAction;
22+
import com.webank.wedatasphere.dss.standard.app.development.listener.common.RefExecutionState;
23+
import org.apache.log4j.Logger;
24+
25+
import java.util.Properties;
26+
27+
public class DataChecker {
28+
public final static String SOURCE_TYPE = "source.type";
29+
public final static String DATA_OBJECT = "check.object";
30+
public final static String WAIT_TIME = "max.check.hours";
31+
public final static String QUERY_FREQUENCY = "query.frequency";
32+
public final static String TIME_SCAPE = "time.scape";
33+
public final static String MASK_URL = "bdp.mask.url";
34+
public final static String MASK_APP_ID = "bdp.mask.app.id";
35+
public final static String MASK_APP_TOKEN = "bdp.mask.app.token";
36+
37+
private Properties p;
38+
private static final Logger logger = Logger.getRootLogger();
39+
DataCheckerDao wbDao = DataCheckerDao.getInstance();
40+
DataCheckerExecutionAction dataCheckerAction = null;
41+
public long maxWaitTime;
42+
public int queryFrequency;
43+
44+
public DataChecker(Properties p, DataCheckerExecutionAction action) {
45+
this.p = p;
46+
dataCheckerAction = action;
47+
maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 3600 * 1000;
48+
//test over time
49+
// maxWaitTime = Long.valueOf(p.getProperty(DataChecker.WAIT_TIME, "1")) * 120 * 1000;
50+
queryFrequency = Integer.valueOf(p.getProperty(DataChecker.QUERY_FREQUENCY, "30000"));
51+
52+
}
53+
54+
public void run() {
55+
dataCheckerAction.setState(RefExecutionState.Running);
56+
try {
57+
if(p == null) {
58+
throw new RuntimeException("Properties is null. Can't continue");
59+
}
60+
if (!p.containsKey(SOURCE_TYPE)) {
61+
logger.info("Properties " + SOURCE_TYPE + " value is Null !");
62+
}
63+
if (!p.containsKey(DATA_OBJECT)) {
64+
logger.info("Properties " + DATA_OBJECT + " value is Null !");
65+
}
66+
begineCheck(dataCheckerAction);
67+
}catch (Exception ex){
68+
dataCheckerAction.setState(RefExecutionState.Failed);
69+
throw new RuntimeException("get DataChecker result failed", ex);
70+
}
71+
72+
}
73+
74+
public void begineCheck(RefExecutionAction action){
75+
boolean success=false;
76+
try {
77+
success= wbDao.validateTableStatusFunction(p, logger,action);
78+
}catch (Exception ex){
79+
dataCheckerAction.setState(RefExecutionState.Failed);
80+
logger.error("datacheck error",ex);
81+
throw new RuntimeException("get DataChecker result failed", ex);
82+
}
83+
if(success) {
84+
dataCheckerAction.setState(RefExecutionState.Success);
85+
}else {
86+
dataCheckerAction.setState(RefExecutionState.Running);
87+
}
88+
}
89+
90+
public void cancel() {
91+
}
92+
93+
}

‎dss-appconn/appconns/dss-datachecker-appconn/src/main/java/com/webank/wedatapshere/dss/appconn/datachecker/DataCheckerAppConn.java‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
/*
2+
* Copyright 2019 WeBank
3+
* Licensed under the Apache License, Version 2.0 (the "License");
4+
* you may not use this file except in compliance with the License.
5+
* You may obtain a copy of the License at
6+
*
7+
* http://www.apache.org/licenses/LICENSE-2.0
8+
*
9+
* Unless required by applicable law or agreed to in writing, software
10+
* distributed under the License is distributed on an "AS IS" BASIS,
11+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
* See the License for the specific language governing permissions and
13+
* limitations under the License.
14+
*
15+
*/
16+
17+
package com.webank.wedatapshere.dss.appconn.datachecker;
18+
19+
import com.webank.wedatapshere.dss.appconn.datachecker.standard.DataCheckerDevelopmentStandard;
20+
import com.webank.wedatasphere.dss.appconn.core.ext.OnlyDevelopmentAppConn;
21+
import com.webank.wedatasphere.dss.appconn.core.impl.AbstractAppConn;
22+
import com.webank.wedatasphere.dss.standard.app.development.standard.DevelopmentIntegrationStandard;
23+
24+
public class DataCheckerAppConn extends AbstractAppConn implements OnlyDevelopmentAppConn {
25+
26+
private DataCheckerDevelopmentStandard standard;
27+
28+
@Override
29+
protected void initialize() {
30+
standard = new DataCheckerDevelopmentStandard();
31+
}
32+
33+
@Override
34+
public DevelopmentIntegrationStandard getOrCreateDevelopmentStandard() {
35+
return standard;
36+
}
37+
}

0 commit comments

Comments
 (0)