Merge pull request #952 from Zosimer/dss-datamodel

Linkis1.2.0[datamodel][dataassets][datawarehouse]

Author: zqburde

assembly/bin/executeSQL.sh

@@ -69,6 +69,10 @@ function executeSQL() {
   isSuccess "source dss_dataapi_ddl.sql"
   mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/dss_guide_ddl.sql"
   isSuccess "source dss_guide_ddl.sql"
+  mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/datamodel.sql"
+  isSuccess "source datamodel.sql"
+  mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_CONF_PATH/apps/datawarehouse.sql"
+  isSuccess "source datawarehouse.sql"
   mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source $DB_DML_PATH"
   isSuccess "source dss_dml_real.sql"
   echo "Rebuild the table"

assembly/bin/install.sh

@@ -121,11 +121,18 @@ function replaceCommonIp() {
     DSS_DATA_API_SERVER_PORT=9208
   fi
 
-  if [ -z "$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP" ]; then
-    DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=$LOCAL_IP
+  if [ -z "$DSS_DATA_ASSETS_SERVER_INSTALL_IP" ]; then
+    DSS_DATA_ASSETS_SERVER_INSTALL_IP=$LOCAL_IP
   fi
-  if [ -z "$DSS_DATA_GOVERNANCE_SERVER_PORT" ]; then
-    DSS_DATA_GOVERNANCE_SERVER_PORT=9209
+  if [ -z "$DSS_DATA_ASSETS_SERVER_INSTALL_IP" ]; then
+    DSS_DATA_ASSETS_SERVER_INSTALL_IP=9300
+  fi
+ if [ -z "$DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP" ]; then
+    DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP=9400
+  fi
+
+ if [ -z "$DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP" ]; then
+    DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP=9500
   fi
 
   if [ -z "$DSS_GUIDE_SERVER_INSTALL_IP" ]; then
@@ -343,13 +350,38 @@ function installDssProject() {
   CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
   installPackage
 
-  ###install dss-data-governance-server
-  SERVER_NAME=dss-data-governance-server
-  SERVER_IP=$DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP
-  SERVER_PORT=$DSS_DATA_GOVERNANCE_SERVER_PORT
-  UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-governance/$SERVER_NAME
-  LIB_PATH=$SERVER_HOME/lib/dss-data-governance
-  LOG_PATH=$SERVER_HOME/logs/dss-data-governance/$SERVER_NAME
+  ###install dss-data-assets-server
+  SERVER_NAME=dss-data-assets-server
+  SERVER_IP=$DSS_DATA_ASSETS_SERVER_INSTALL_IP
+  SERVER_PORT=$DSS_DATA_ASSETS_SERVER_PORT
+  UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-assets-server/$SERVER_NAME
+  LIB_PATH=$SERVER_HOME/lib/dss-data-assets-server
+  LIB_PATH=$SERVER_HOME/lib/atlas-application.properties
+  LOG_PATH=$SERVER_HOME/logs/dss-data-assets-server/$SERVER_NAME
+  CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
+  CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
+  CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
+  installPackage
+
+  ###install dss-data-warehouse-server
+  SERVER_NAME=dss-data-warehouse-server
+  SERVER_IP=$DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP
+  SERVER_PORT=$DSS_DATA_WAREHOUSE_SERVER_PORT
+  UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-data-warehouse-server/$SERVER_NAME
+  LIB_PATH=$SERVER_HOME/lib/dss-data-warehouse-server
+  LOG_PATH=$SERVER_HOME/logs/dss-data-warehouse-server/$SERVER_NAME
+  CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
+  CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
+  CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml
+  installPackage
+
+  ###install dss-datamodel-center-server
+  SERVER_NAME=dss-datamodel-center-server
+  SERVER_IP=$DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP
+  SERVER_PORT=$DSS_DATAMODEL_CENTER_SERVER_PORT
+  UPLOAD_LIB_FILES=$DSS_FILE_PATH/lib/dss-datamodel-center-server/$SERVER_NAME
+  LIB_PATH=$SERVER_HOME/lib/dss-datamodel-center-server
+  LOG_PATH=$SERVER_HOME/logs/dss-datamodel-center-server/$SERVER_NAME
   CONF_SERVER_PROPERTIES=$SERVER_HOME/conf/$SERVER_NAME.properties
   CONF_DSS_PROPERTIES=$SERVER_HOME/conf/dss.properties
   CONF_APPLICATION_YML=$SERVER_HOME/conf/application-dss.yml

assembly/config/config.sh

@@ -52,13 +52,19 @@ DSS_FLOW_EXECUTION_SERVER_PORT=9006
 ###dss-scriptis-server
 DSS_SCRIPTIS_SERVER_INSTALL_IP=127.0.0.1
 DSS_SCRIPTIS_SERVER_PORT=9008
+###dss-data-assets-server
+DSS_DATA_ASSETS_SERVER_INSTALL_IP=127.0.0.1
+DSS_DATA_ASSETS_SERVER_PORT=9300
+###dss-datamodel-center-server
+DSS_DATAMODEL_CENTER_SERVER_INSTALL_IP=127.0.0.1
+DSS_DATAMODEL_CENTER_SERVER_PORT=9400
+###dss-data-warehouse-server
+DSS_DATA_WAREHOUSE_SERVER_INSTALL_IP=127.0.0.1
+DSS_DATA_WAREHOUSE_SERVER_PORT=9500
 
 ###dss-data-api-server
 DSS_DATA_API_SERVER_INSTALL_IP=127.0.0.1
 DSS_DATA_API_SERVER_PORT=9208
-###dss-data-governance-server
-DSS_DATA_GOVERNANCE_SERVER_INSTALL_IP=127.0.0.1
-DSS_DATA_GOVERNANCE_SERVER_PORT=9209
 ###dss-guide-server
 DSS_GUIDE_SERVER_INSTALL_IP=127.0.0.1
 DSS_GUIDE_SERVER_PORT=9210
@@ -83,5 +89,4 @@ EMAIL_PORT=25
 [email protected]
 EMAIL_PASSWORD=xxxxx
 EMAIL_PROTOCOL=smtp
-############## ############## dss_appconn_instance configuration   end   ############## ##############
-
+############## ############## dss_appconn_instance configuration   end   ############## ##############

assembly/dss-package/pom.xml

@@ -21,7 +21,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>1.1.0</version>
+        <version>1.2.0</version>
     </parent>
     <modelVersion>4.0.0</modelVersion>
     <artifactId>dss-package</artifactId>

assembly/dss-package/src/main/assembly/distribution.xml

@@ -287,20 +287,40 @@
     <!--dss-data-api-server-->
     <fileSet>
       <directory>
-        ${basedir}/../../dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/
+        ${basedir}/../../dss-apps/dss-data-api/dss-data-api-server/target/out/dss-data-api-server/lib/
       </directory>
-      <outputDirectory>lib/dss-data-api/dss-data-api-server</outputDirectory>
+      <outputDirectory>lib/dss-apps/dss-data-api-server</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
+
+
+    <fileSet>
+      <directory>
+        ${basedir}/../../dss-apps/dss-dataasset-management/dss-data-assets-server/target/out/dss-data-assets-server/lib/
+      </directory>
+      <outputDirectory>lib/dss-apps/dss-data-assets-server</outputDirectory>
+      <includes>
+        <include>**/*</include>
+      </includes>
+    </fileSet>
+
+    <fileSet>
+      <directory>
+        ${basedir}/../../dss-apps/dss-datamodel-center/dss-datamodel-center-server/target/out/dss-datamodel-center-server/lib/
+      </directory>
+      <outputDirectory>lib/dss-apps/dss-datamodel-center-server</outputDirectory>
       <includes>
         <include>**/*</include>
       </includes>
     </fileSet>
 
-    <!--dss-data-governance-server-->
     <fileSet>
       <directory>
-        ${basedir}/../../dss-data-governance/dss-data-governance-server/target/out/dss-data-governance-server/lib/
+        ${basedir}/../../dss-apps/dss-datawarehouse-design/dss-data-warehouse-server/target/out/dss-data-warehouse-server/lib/
       </directory>
-      <outputDirectory>lib/dss-data-governance/dss-data-governance-server</outputDirectory>
+      <outputDirectory>lib/dss-apps/dss-data-warehouse-server</outputDirectory>
       <includes>
         <include>**/*</include>
       </includes>

assembly/pom.xml

@@ -22,7 +22,7 @@
     <parent>
         <artifactId>dss</artifactId>
         <groupId>com.webank.wedatasphere.dss</groupId>
-        <version>1.1.0</version>
+        <version>1.2.0</version>
     </parent>
     <packaging>pom</packaging>
     <modelVersion>4.0.0</modelVersion>

conf/atlas-application.properties

@@ -0,0 +1,67 @@
+# Generated by Apache Ambari. Sun Feb 27 10:44:18 2022
+
+atlas.audit.hbase.tablename=ATLAS_ENTITY_AUDIT_EVENTS
+atlas.audit.hbase.zookeeper.quorum=hdp
+atlas.audit.zookeeper.session.timeout.ms=60000
+atlas.authentication.keytab=/etc/security/keytabs/atlas.service.keytab
+atlas.authentication.method.file=true
+atlas.authentication.method.file.filename=/home/hadoop/software/apache-atlas-2.0.0/conf/users-credentials.properties
+atlas.authentication.method.kerberos=false
+atlas.authentication.method.ldap=false
+atlas.authentication.method.ldap.ad.base.dn=
+atlas.authentication.method.ldap.ad.bind.dn=
+atlas.authentication.method.ldap.ad.bind.password=
+atlas.authentication.method.ldap.ad.default.role=ROLE_USER
+atlas.authentication.method.ldap.ad.domain=
+atlas.authentication.method.ldap.ad.referral=ignore
+atlas.authentication.method.ldap.ad.url=
+atlas.authentication.method.ldap.ad.user.searchfilter=(sAMAccountName={0})
+atlas.authentication.method.ldap.base.dn=
+atlas.authentication.method.ldap.bind.dn=
+atlas.authentication.method.ldap.bind.password=
+atlas.authentication.method.ldap.default.role=ROLE_USER
+atlas.authentication.method.ldap.groupRoleAttribute=cn
+atlas.authentication.method.ldap.groupSearchBase=
+atlas.authentication.method.ldap.groupSearchFilter=
+atlas.authentication.method.ldap.referral=ignore
+atlas.authentication.method.ldap.type=ldap
+atlas.authentication.method.ldap.url=
+atlas.authentication.method.ldap.user.searchfilter=
+atlas.authentication.method.ldap.userDNpattern=uid=
+atlas.authentication.principal=atlas
+atlas.authorizer.impl=ranger
+atlas.cluster.name=bigdata_all_in_one
+atlas.enableTLS=false
+atlas.graph.index.search.solr.mode=cloud
+atlas.graph.index.search.solr.wait-searcher=true
+atlas.graph.index.search.solr.zookeeper-url=hadoop:2181/infra-solr
+atlas.graph.storage.hbase.table=atlas_janus
+atlas.graph.storage.hostname=hdp
+atlas.kafka.auto.commit.enable=false
+atlas.kafka.bootstrap.servers=hadoop:6667
+atlas.kafka.hook.group.id=atlas
+atlas.kafka.zookeeper.connect=hadoop:2181
+atlas.kafka.zookeeper.connection.timeout.ms=30000
+atlas.kafka.zookeeper.session.timeout.ms=60000
+atlas.kafka.zookeeper.sync.time.ms=20
+atlas.lineage.schema.query.hive_table=hive_table where __guid='%s'\, columns
+atlas.lineage.schema.query.Table=Table where __guid='%s'\, columns
+atlas.notification.create.topics=true
+atlas.notification.embedded=false
+atlas.notification.replicas=1
+atlas.notification.topics=ATLAS_HOOK,ATLAS_ENTITIES
+atlas.proxyusers=
+atlas.rest.address=http://hadoop:21000
+atlas.server.address.id1=hadoop:21000
+atlas.server.bind.address=0.0.0.0
+atlas.server.ha.enabled=false
+atlas.server.http.port=21000
+atlas.server.https.port=21443
+atlas.server.ids=id1
+atlas.simple.authz.policy.file=/home/hadoop/software/apache-atlas-2.0.0/conf/atlas-simple-authz-policy.json
+atlas.solr.kerberos.enable=false
+atlas.ssl.exclude.protocols=TLSv1.2
+atlas.sso.knox.browser.useragent=
+atlas.sso.knox.enabled=false
+atlas.sso.knox.providerurl=
+atlas.sso.knox.publicKey=

conf/dss-data-assets-server.properties

@@ -0,0 +1,88 @@
+#
+# /*
+#  * Copyright 2019 WeBank
+#  *
+#  * Licensed under the Apache License, Version 2.0 (the "License");
+#  *  you may not use this file except in compliance with the License.
+#  * You may obtain a copy of the License at
+#  *
+#  * http://www.apache.org/licenses/LICENSE-2.0
+#  *
+#  * Unless required by applicable law or agreed to in writing, software
+#  * distributed under the License is distributed on an "AS IS" BASIS,
+#  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  * See the License for the specific language governing permissions and
+#  * limitations under the License.
+#  */
+#
+# Spring configurations
+spring.server.port=9300
+spring.spring.application.name=dss-data-assets-server
+wds.linkis.test.mode=true
+wds.linkis.log.clear=true
+wds.linkis.server.version=v1
+
+
+wds.linkis.server.mybatis.datasource.url=jdbc:mysql://localhost:3306/hive?useSSL=false&characterEncoding=UTF-8
+
+wds.linkis.server.mybatis.datasource.username=root
+
+***REMOVED***root
+
+
+
+
+
+wds.linkis.server.restful.scan.packages=com.webank.wedatasphere.dss.data.governance.restful
+
+##mybatis
+wds.linkis.server.mybatis.mapperLocations=classpath*:com/webank/wedatasphere/dss/data/governance/dao/impl/*.xml
+
+wds.linkis.server.mybatis.typeAliasesPackage=com.webank.wedatasphere.dss.data.governance.entity
+
+wds.linkis.server.mybatis.BasePackage=com.webank.wedatasphere.dss.data.governance.dao
+
+
+# atlas config
+atlas.rest.address=http://localhost:21000
+atlas.username=admin
+atlas.password=admin123
+atlas.client.readTimeoutMSecs=60000
+atlas.client.connectTimeoutMSecs=60000
+
+# create root classification
+atlas.root.indicator=datamodel_indicator
+atlas.root.measure=datamodel_measure
+atlas.root.dimension=datamodel_dimension
+atlas.root.layer=datamodel_layer
+atlas.root.theme=datamodel_theme
+
+# create root glossary
+atlas.root.label=datamodel_label
+atlas.root.collection=datamodel_collection
+
+
+# hive metadata config
+metastore.datasource.driver=com.mysql.jdbc.Driver
+metastore.datasource.url=jdbc:mysql://localhost:3306/hive?characterEncoding=UTF-8
+metastore.datasource.username=root
+metastore.datasource.password=root
+
+
+wds.workspace.client.serverurl=http://localhost:9001
+
+wds.workspace.client.authtoken.key=hadoop
+
+wds.workspace.client.authenticationStrategy=org.apache.linkis.httpclient.dws.authentication.TokenAuthenticationStrategy
+
+wds.workspace.client.discovery.enabled=false
+
+wds.workspace.client.authtoken.value=hadoop
+
+spring.spring.mvc.servlet.path=/api/rest_j/v1
+
+wds.workspace.client.dws.version=v1
+
+wds.wedatasphere.data.assert.client.strategy=static
+
+