merge conflicts & product iterate

Author: zhangxiang

.gitignore

@@ -6,6 +6,10 @@
 assembly/target
 assembly/dss-package/target
 
+
+# dss-commons
+dss-commons/dss-sender-service/target
+
 # dss-appconn
 dss-appconn/appconns/dss-datachecker-appconn/target
 dss-appconn/appconns/dss-eventchecker-appconn/target
@@ -20,7 +24,9 @@ dss-appconn/dss-appconn-loader/target
 dss-appconn/dss-schedule-appconn-core/target
 dss-appconn/linkis-appconn-engineplugin/target
 dss-appconn/appconns/dss-dolphinscheduler-appconn/target
-
+dss-appconn/dss-appconn-manager/dss-appconn-manager-client/target
+dss-appconn/dss-appconn-manager/dss-appconn-manager-core/target
+dss-appconn/dss-scheduler-appconn/target
 
 # dss-apps
 dss-apps/dss-apiservice-server/target
@@ -51,6 +57,9 @@ dss-orchestrator/orchestrators/dss-workflow/dss-flow-execution-server/target
 dss-orchestrator/orchestrators/dss-workflow/dss-linkis-node-execution/target
 dss-orchestrator/orchestrators/dss-workflow/dss-workflow-common/target
 dss-orchestrator/orchestrators/dss-workflow/dss-workflow-server/target
+dss-orchestrator/dss-orchestrator-conversion-standard/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-conversion-standard/target
+dss-orchestrator/orchestrators/dss-workflow/dss-workflow-sdk/target
 
 # dss-standars
 dss-standard/dss-standard-common/target
@@ -63,10 +72,26 @@ dss-standard/structure-standard/dss-project-plugin/target
 dss-standard/structure-standard/dss-role-plugin/target
 dss-standard/structure-standard/dss-structure-integration-standard
 dss-standard/structure-standard/spring-origin-dss-project-plugin/target
+
 #dss-data-api
+dss-data-api/dss-data-api-server/src/main/resources
 dss-data-api/dss-api-sql-template/target
 dss-data-api/dss-data-api-server/target
 
+#dss-data-governance
+dss-data-governance/dss-data-governance-server/src/main/resources
+dss-data-governance/dss-data-governance-server/target
+dss-data-governance/dss-data-governance-common/target
+dss-data-governance/dss-data-asset-server/target
+dss-data-governance/dss-data-classification-server/target
+dss-data-governance/dss-data-warehouse-dao/target
+dss-data-governance/dss-data-warehouse-service/target
+dss-data-governance/dss-data-warehouse-server/target
+
+#dss-guide
+dss-guide/dss-guide-server/src/main/resources/
+dss-guide/dss-guide-server/target
+
 
 # plugins
 plugins/azkaban/linkis-jobtype/target
@@ -77,5 +102,3 @@ logs/*
 
 *.iml
 /plugins/dolphinscheduler/linkis-dolphinscheduler-client/target/
-
-web/yarn.lock

.vscode/settings.json

@@ -0,0 +1,4 @@
+{
+  "editor.formatOnPaste": true,
+  "editor.formatOnType": true
+}

README-ZH.md

@@ -33,75 +33,77 @@ Please be patient, it will take some time to load gif.
 
         c. [Qualitis](https://github.com/WeBankFinTech/Qualitis) - Data Quality Management Tool
 
-        d. [Azkaban](https://azkaban.github.io/) - Batch workflow job scheduler
+        d. [Schedulis](https://github.com/WeBankFinTech/Schedulis) - Batch workflow job scheduler
+
+        f. [Exchangis](https://github.com/WeBankFinTech/Exchangis) - Data Exchange Tool
 
 ![DSS one-stop video](images/en_US/readme/onestop.gif) 
 
-### 2. AppJoint, based on Linkis，defines a unique design concept
+### 2. AppConn, based on Linkis，defines a unique design concept
 
-        AppJoint——application joint, defining unified front-end and back-end 
+        AppConn——application connector, defining unified front-end and back-end 
                             integration specifications, can quickly and easily integrate with external data application systems, 
                             making them as part of DSS data application development.
 
-        DSS arranges multiple AppJoints in series to form a workflow that supports real-time execution and scheduled execution. Users can complete the entire process development of data applications with simple drag and drop operations.
+        DSS arranges multiple AppConns in series to form a workflow that supports real-time execution and scheduled execution. Users can complete the entire process development of data applications with simple drag and drop operations.
 
-        Since AppJoint is integrated with Linkis, the external data application system shares the capabilities of resource management, concurrent limiting, and high performance. AppJoint also allows sharable context across system level and completely gets away from application silos.
+        Since AppConn is integrated with Linkis, the external data application system shares the capabilities of resource management, concurrent limiting, and high performance. AppConn also allows sharable context across system level and completely gets away from application silos.
 
 ### 3. Project, as the management unit
 
         With Project as the management unit, DSS organizes and manages the business applications of each data application system, and defines a set of common standards for collaborative development of projects across data application systems.
 
 ### 4. Integrated data application components
 
-       a. Azkaban AppJoint —— Batch workflow job scheduler
+       a. Schedulis AppConn —— Batch workflow job scheduler
 
          Many data applications developed by users usually require periodic scheduling capability.
 
          At present, the open source scheduling system in the community is pretty unfriendly to integrate with other data application systems.
 
-         DSS implements Azkaban AppJoint, which allows users to publish DSS workflows to Azkaban for regular scheduling.
+         DSS implements Schedulis AppConn, which allows users to publish DSS workflows to Azkaban for regular scheduling.
 
          DSS also defines standard and generic workflow parsing and publishing specifications for scheduling systems, allowing other scheduling systems to easily achieve low-cost integration with DSS.
 
 ![Azkaban](images/en_US/readme/Azkaban_AppJoint.gif)
 
-       b. Scriptis AppJoint —— Data Development IDE Tool
+       b. Scriptis AppConn —— Data Development IDE Tool
 
          What is [Scriptis](https://github.com/WeBankFinTech/Scriptis)?
 
          Scriptis is for interactive data analysis with script development(SQL, Pyspark, HiveQL), task submission(Spark, Hive), UDF, function, resource management and intelligent diagnosis.
 
-         Scriptis AppJoint integrates the data development capabilities of Scriptis to DSS, and allows various script types of Scriptis to serve as nodes in the DSS workflow to participate in the application development process.
+         Scriptis AppConn integrates the data development capabilities of Scriptis to DSS, and allows various script types of Scriptis to serve as nodes in the DSS workflow to participate in the application development process.
 
          Currently supports HiveSQL, SparkSQL, Pyspark, Scala and other script node types.
 
 ![Scriptis](images/en_US/readme/Scriptis_AppJoint.gif)
 
-       c. Visualis AppJoint —— Data Visualization Tool
+       c. Visualis AppConn —— Data Visualization Tool
 
          What is [Visualis](https://github.com/WeBankFinTech/Visualis)?
 
          Visualis is a BI tool for data visualization. It provides financial-grade data visualization capabilities on the basis of data security and permissions, based on the open source project Davinci contributed by CreditEase.
 
-         Visualis AppJoint integrates data visualization capabilities to DSS, and allows displays and dashboards, as nodes of DSS workflows, to be associated with upstream data market.
+         Visualis AppConn integrates data visualization capabilities to DSS, and allows displays and dashboards, as nodes of DSS workflows, to be associated with upstream data market.
 
 ![Visualis](images/en_US/readme/Visualis_AppJoint.gif)
 
-       d. Qualitis AppJoint —— Data quality management Tool
+       d. Qualitis AppConn —— Data quality management Tool
 
-         Qualitis AppJoint integrates data quality verification capabilities for DSS, allows Qualitis as a node in DSS workflow
+         Qualitis AppConn integrates data quality verification capabilities for DSS, allows Qualitis as a node in DSS workflow
 
 ![Qualitis](images/en_US/readme/Qualitis_AppJoint.gif)
 
-       e. Data Sender——Sender AppJoint
+       e. Data Sender——Sender AppConn
 
-         Sender AppJoint provides data delivery capability for DSS. Currently it supports the SendEmail node type, and the result sets of all other nodes can be sent via email.
+         Sender AppConn provides data delivery capability for DSS. Currently it supports the SendEmail node type, and the result sets of all other nodes can be sent via email.
 
          For example, the SendEmail node can directly send the screen shot of a display as an email.  
 
-       f. Signal AppJoint —— Signal Nodes
+       f. Signal AppConn —— Signal Nodes
 
-         Signal AppJoint is used to strengthen the correlation between business and process while keeping them decoupled.
+         Signal AppConn is used to strengthen the correlation between business and process while keeping them decoupled.
 
          DataChecker Node：Checks whether a table or partition exists.
 

README.md

@@ -0,0 +1,109 @@
+#!/bin/sh
+#Actively load user env
+source ~/.bashrc
+shellDir=`dirname $0`
+workDir=`cd ${shellDir}/..;pwd`
+
+SOURCE_ROOT=${workDir}
+
+#load config
+source ${SOURCE_ROOT}/conf/config.sh
+source ${SOURCE_ROOT}/conf/db.sh
+
+APPCONN_NAME=''
+APPCONN_INSTALL_IP=127.0.0.1
+APPCONN_INSTALL_PORT=8088
+
+#echo "Current path of init sql is ${DB_DML_PATH}"
+LOCAL_IP="`ifconfig | grep 'inet' | grep -v '127.0.0.1' | cut -d: -f2 | awk '{ print $2}'`"
+
+function isSuccess(){
+  if [ $? -ne 0 ]; then
+      echo "Failed to " + $1
+      exit 1
+  else
+      echo "Succeed to" + $1
+  fi
+}
+
+PROC_NAME=DSSProjectServerApplication
+ProcNumber=`ps -ef |grep -w $PROC_NAME|grep -v grep|wc -l`
+if [ $ProcNumber -le 0 ];then
+   echo "${PROC_NAME} is not running,Please check whether DSS is installed"
+   exit 1000
+else
+   echo "Begine to install appconn"
+fi
+
+##choose install mysql mode
+function initInstallAppConn() {
+  echo "Please select the type of installation component?"
+  echo " 1: schedulis"
+  echo " 2: visualis"
+  echo " 3：Your AppConn Name"
+  echo " 4：exit"
+  read -p "Please input the choice:"  idx
+  if [[ '1' = "$idx" ]];then
+   APPCONN_NAME="schedulis"
+  elif [[ '2' = "$idx" ]];then
+    APPCONN_NAME="visualis"
+  elif [[ '4' = "$idx" ]];then
+    echo "no choice,exit!"
+    exit 1
+  else
+     APPCONN_NAME=$idx
+  fi
+  echo "Current installation component is ${APPCONN_NAME}"
+
+  echo ""
+  echo "If this machine(127.0.0.1) is installed, enter 1"
+  echo "For others, you need to enter a complete IP address."
+  read -p "Please enter the ip of appconn: "  ip
+  APPCONN_INSTALL_IP=$ip
+  if [[ '1' = "$ip" ]];then
+   APPCONN_INSTALL_IP="127.0.0.1"
+  fi
+  echo "You input ip is ${APPCONN_INSTALL_IP}"
+
+  echo ""
+  read -p "Please enter the port of appconn:"  port
+  APPCONN_INSTALL_PORT=$port
+  echo "You input ip is ${APPCONN_INSTALL_PORT}"
+}
+
+function replaceCommonIp() {
+ if [[ $APPCONN_INSTALL_IP == "127.0.0.1" ]] || [[ $APPCONN_INSTALL_IP == "0.0.0.0" ]];then
+    echo "APPCONN_INSTALL_IP is equals $APPCONN_INSTALL_IP, we will change it to ip address"
+    APPCONN_INSTALL_IP=$LOCAL_IP
+ fi
+}
+
+##choose execute mysql mode
+function executeSQL() {
+  TEMP_DB_DML_PATH=${SOURCE_ROOT}/dss-appconns/${APPCONN_NAME}/db
+  DB_DML_PATH=$TEMP_DB_DML_PATH/init_real.sql
+  cp -rf $TEMP_DB_DML_PATH/init.sql $DB_DML_PATH
+  sed -i "s/APPCONN_INSTALL_IP/$APPCONN_INSTALL_IP/g"       $DB_DML_PATH
+  sed -i "s/APPCONN_INSTALL_PORT/$APPCONN_INSTALL_PORT/g"   $DB_DML_PATH
+  sed -i "s#DSS_INSTALL_HOME_VAL#$DSS_INSTALL_HOME#g" $DB_DML_PATH
+  mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
+  isSuccess "source $DB_DML_PATH"
+  echo "the table update finished"
+}
+
+echo ""
+echo "step1:Initialize installation settings"
+initInstallAppConn
+echo ""
+
+echo "step2:replaceIp"
+replaceCommonIp
+echo ""
+
+echo "step3:update database"
+executeSQL
+echo ""
+
+echo "step4:refresh appconn load"
+curl -H "Token-Code:BML-AUTH" -H "Token-User:hadoop" -X GET http://${GATEWAY_INSTALL_IP}:${GATEWAY_PORT}/api/rest_j/v1/dss/framework/project/appconn/${APPCONN_NAME}/load
+echo ""

assembly/bin/appconn-install.sh

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Copyright 2019 WeBank
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+say() {
+    printf 'check command fail \n %s\n' "$1"
+}
+
+err() {
+    say "$1" >&2
+    exit 1
+}
+
+check_cmd() {
+    command -v "$1" > /dev/null 2>&1
+}
+
+need_cmd() {
+    if ! check_cmd "$1"; then
+        err "need '$1' (your linux command not found)"
+    fi
+}
+echo "<-----start to check used cmd---->"
+need_cmd yum
+need_cmd java
+need_cmd mysql
+need_cmd telnet
+need_cmd tar
+need_cmd sed
+need_cmd dos2unix
+echo "<-----end to check used cmd---->"