adapt for dynamically add/remove graph (#138)

Change-Id: I005fba5a131f55820dbd1d702eb5ef812acd132f
Co-authored-by: Jermy Li <javaloveme@gmail.com>

Author: zhoney

.github/workflows/ci.yml

@@ -17,7 +17,7 @@ jobs:
     runs-on: ubuntu-20.04
     env:
       TRAVIS_DIR: assembly/travis
-      COMMIT_ID: 2dd2d7c3fd7cb42cd7338d70a73e04b0c5604487
+      COMMIT_ID: 1d031c5905cbef008dd5fb468576b0e6a9445181
     steps:
       - name: Install JDK 8
         uses: actions/setup-java@v2
@@ -43,7 +43,7 @@ jobs:
 
       - name: Prepare env and service
         run: |
-          $TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID | grep -v "Downloading\|Downloaded"
+          $TRAVIS_DIR/install-hugegraph-from-source.sh $COMMIT_ID
 
       - name: Run test
         run: |

BCLOUD

@@ -6,8 +6,8 @@
 scriptEvaluationTimeout: 30000
 
 channelizer: org.apache.tinkerpop.gremlin.server.channel.WsAndHttpChannelizer
+# don't set graph at here, this happens after support for dynamically adding graph
 graphs: {
-  hugegraph: conf/hugegraph.properties
 }
 scriptEngines: {
   gremlin-groovy: {
@@ -17,16 +17,30 @@ scriptEngines: {
       org.apache.tinkerpop.gremlin.jsr223.ImportGremlinPlugin: {
         classImports: [
           java.lang.Math,
+          com.baidu.hugegraph.backend.id.IdGenerator,
           com.baidu.hugegraph.type.define.Directions,
-          com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
+          com.baidu.hugegraph.type.define.NodeRole,
+          com.baidu.hugegraph.traversal.algorithm.CollectionPathsTraverser,
+          com.baidu.hugegraph.traversal.algorithm.CountTraverser,
           com.baidu.hugegraph.traversal.algorithm.CustomizedCrosspointsTraverser,
+          com.baidu.hugegraph.traversal.algorithm.CustomizePathsTraverser,
           com.baidu.hugegraph.traversal.algorithm.FusiformSimilarityTraverser,
           com.baidu.hugegraph.traversal.algorithm.HugeTraverser,
+          com.baidu.hugegraph.traversal.algorithm.JaccardSimilarTraverser,
+          com.baidu.hugegraph.traversal.algorithm.KneighborTraverser,
+          com.baidu.hugegraph.traversal.algorithm.KoutTraverser,
+          com.baidu.hugegraph.traversal.algorithm.MultiNodeShortestPathTraverser,
           com.baidu.hugegraph.traversal.algorithm.NeighborRankTraverser,
           com.baidu.hugegraph.traversal.algorithm.PathsTraverser,
           com.baidu.hugegraph.traversal.algorithm.PersonalRankTraverser,
+          com.baidu.hugegraph.traversal.algorithm.SameNeighborTraverser,
           com.baidu.hugegraph.traversal.algorithm.ShortestPathTraverser,
+          com.baidu.hugegraph.traversal.algorithm.SingleSourceShortestPathTraverser,
           com.baidu.hugegraph.traversal.algorithm.SubGraphTraverser,
+          com.baidu.hugegraph.traversal.algorithm.TemplatePathsTraverser,
+          com.baidu.hugegraph.traversal.algorithm.steps.EdgeStep,
+          com.baidu.hugegraph.traversal.algorithm.steps.RepeatEdgeStep,
+          com.baidu.hugegraph.traversal.algorithm.steps.WeightedEdgeStep,
           com.baidu.hugegraph.traversal.optimize.Text,
           com.baidu.hugegraph.traversal.optimize.TraversalUtil,
           com.baidu.hugegraph.util.DateUtil
@@ -40,12 +54,6 @@ scriptEngines: {
   }
 }
 serializers:
-  - { className: org.apache.tinkerpop.gremlin.driver.ser.GryoLiteMessageSerializerV1d0,
-      config: {
-        serializeResultToString: false,
-        ioRegistries: [com.baidu.hugegraph.io.HugeGraphIoRegistry]
-      }
-  }
   - { className: org.apache.tinkerpop.gremlin.driver.ser.GraphBinaryMessageSerializerV1,
       config: {
         serializeResultToString: false,
@@ -72,7 +80,7 @@ serializers:
   }
 metrics: {
   consoleReporter: {enabled: false, interval: 180000},
-  csvReporter: {enabled: true, interval: 180000, fileName: /tmp/gremlin-server-metrics.csv},
+  csvReporter: {enabled: false, interval: 180000, fileName: ./metrics/gremlin-server-metrics.csv},
   jmxReporter: {enabled: false},
   slf4jReporter: {enabled: false, interval: 180000},
   gangliaReporter: {enabled: false, interval: 180000, addressingMode: MULTICAST},

assembly/travis/conf/hugegraph.properties …/travis/conf/graphs/hugegraph.propertiesassembly/travis/conf/hugegraph.properties renamed to assembly/travis/conf/graphs/hugegraph.properties assembly/travis/conf/hugegraph.properties renamed to assembly/travis/conf/graphs/hugegraph.properties

@@ -3,10 +3,39 @@ restserver.url=http://127.0.0.1:8080
 # gremlin server url, need to be consistent with host and port in gremlin-server.yaml
 #gremlinserver.url=http://127.0.0.1:8182
 
-# graphs list with pair NAME:CONF_PATH
-graphs=[hugegraph:conf/hugegraph.properties]
+graphs=./conf/graphs
 
-# authentication
+# The maximum thread ratio for batch writing, only take effect if the batch.max_write_threads is 0
+batch.max_write_ratio=80
+batch.max_write_threads=0
+
+# authentication configs
+# choose 'com.baidu.hugegraph.auth.StandardAuthenticator' or 'com.baidu.hugegraph.auth.ConfigAuthenticator'
 auth.authenticator=com.baidu.hugegraph.auth.StandardAuthenticator
+
+# for StandardAuthenticator mode
+#auth.graph_store=hugegraph
+# auth client config
+#auth.remote_url=127.0.0.1:8899,127.0.0.1:8898,127.0.0.1:8897
+
+# for ConfigAuthenticator mode
 #auth.admin_token=
 #auth.user_tokens=[]
+
+# rpc group configs of multi graph servers
+# rpc server configs
+rpc.server_host=127.0.0.1
+rpc.server_port=8090
+#rpc.server_timeout=30
+
+# rpc client configs (like enable to keep cache consistency)
+rpc.remote_url=127.0.0.1:8090
+#rpc.client_connect_timeout=20
+#rpc.client_reconnect_period=10
+#rpc.client_read_timeout=40
+#rpc.client_retries=3
+#rpc.client_load_balancer=consistentHash
+
+# lightweight load balancing (beta)
+server.id=server-1
+server.role=master

assembly/travis/conf/gremlin-server.yaml

@@ -9,26 +9,34 @@ fi
 
 COMMIT_ID=$1
 HUGEGRAPH_GIT_URL="https://github.com/hugegraph/hugegraph.git"
+GIT_DIR=hugegraph
 
+# download code and compile
 git clone --depth 100 ${HUGEGRAPH_GIT_URL}
-cd hugegraph
+cd "${GIT_DIR}"
 git checkout ${COMMIT_ID}
 mvn package -DskipTests
-mv hugegraph-*.tar.gz ../
+
+TAR=$(echo hugegraph-*.tar.gz)
+tar -zxvf "${TAR}" -C ../
 cd ../
-rm -rf hugegraph
-tar -zxvf hugegraph-*.tar.gz
+rm -rf "${GIT_DIR}"
 
+HTTP_SERVER_DIR=$(echo hugegraph-*)
 HTTPS_SERVER_DIR="hugegraph_https"
-mkdir ${HTTPS_SERVER_DIR}
-cp -r hugegraph-*/. ${HTTPS_SERVER_DIR}
-cd "$(find hugegraph-* | head -1)"
-cp ../$TRAVIS_DIR/conf/* conf
+
+cp -r "${HTTP_SERVER_DIR}" "${HTTPS_SERVER_DIR}"
+
+# config auth options just for http server (must keep '/.')
+cp -rf "${TRAVIS_DIR}"/conf/. "${HTTP_SERVER_DIR}"/conf/
+
 # start HugeGraphServer with http protocol
+cd "${HTTP_SERVER_DIR}"
 echo -e "pa" | bin/init-store.sh || exit 1
 bin/start-hugegraph.sh || exit 1
 
-cd ../${HTTPS_SERVER_DIR}
+# config options for https server
+cd ../"${HTTPS_SERVER_DIR}"
 REST_SERVER_CONFIG="conf/rest-server.properties"
 GREMLIN_SERVER_CONFIG="conf/gremlin-server.yaml"
 sed -i "s?http://127.0.0.1:8080?https://127.0.0.1:8443?g" "$REST_SERVER_CONFIG"

assembly/travis/conf/rest-server.properties

@@ -5,7 +5,7 @@
 
     <groupId>com.baidu.hugegraph</groupId>
     <artifactId>hugegraph-client</artifactId>
-    <version>2.0.0</version>
+    <version>2.0.1</version>
     <packaging>jar</packaging>
 
     <name>hugegraph-client</name>
@@ -113,7 +113,7 @@
                         <manifestEntries>
                             <!-- Must be on one line, otherwise the automatic
                                  upgrade script cannot replace the version number -->
-                            <Implementation-Version>1.9.4.0</Implementation-Version>
+                            <Implementation-Version>2.0.1.0</Implementation-Version>
                         </manifestEntries>
                     </archive>
                 </configuration>

assembly/travis/install-hugegraph-from-source.sh

@@ -22,6 +22,13 @@
 import java.util.List;
 import java.util.Map;
 
+import javax.ws.rs.core.HttpHeaders;
+import javax.ws.rs.core.MediaType;
+import javax.ws.rs.core.MultivaluedHashMap;
+import javax.ws.rs.core.MultivaluedMap;
+
+import org.apache.commons.lang3.StringUtils;
+
 import com.baidu.hugegraph.api.API;
 import com.baidu.hugegraph.client.RestClient;
 import com.baidu.hugegraph.exception.InvalidResponseException;
@@ -33,6 +40,13 @@
 
 public class GraphsAPI extends API {
 
+    private static final String DELIMITER = "/";
+    private static final String MODE = "mode";
+    private static final String GRAPH_READ_MODE = "graph_read_mode";
+    private static final String CLEAR = "clear";
+
+    private static final String CONFIRM_MESSAGE = "confirm_message";
+
     public GraphsAPI(RestClient client) {
         super(client);
         this.path(this.type());
@@ -43,6 +57,21 @@ protected String type() {
         return HugeType.GRAPHS.string();
     }
 
+    @SuppressWarnings("unchecked")
+    public Map<String, String> create(String name, String cloneGraphName,
+                                      String configText) {
+        this.client.checkApiVersion("0.67", "dynamic graph add");
+        MultivaluedMap<String, Object> headers = new MultivaluedHashMap<>();
+        headers.add(HttpHeaders.CONTENT_TYPE, MediaType.APPLICATION_JSON);
+        Map<String, Object> params = null;
+        if (StringUtils.isNotEmpty(cloneGraphName)) {
+            params = ImmutableMap.of("clone_graph_name", cloneGraphName);
+        }
+        RestResult result = this.client.post(joinPath(this.path(), name),
+                                             configText, headers, params);
+        return result.readObject(Map.class);
+    }
+
     @SuppressWarnings("unchecked")
     public Map<String, String> get(String name) {
         RestResult result = this.client.get(this.path(), name);
@@ -54,19 +83,28 @@ public List<String> list() {
         return result.readList(this.type(), String.class);
     }
 
+    public void clear(String graph, String message) {
+        this.client.delete(joinPath(this.path(), graph, CLEAR),
+                           ImmutableMap.of(CONFIRM_MESSAGE, message));
+    }
+
+    public void drop(String graph, String message) {
+        this.client.checkApiVersion("0.67", "dynamic graph delete");
+        this.client.delete(joinPath(this.path(), graph),
+                           ImmutableMap.of(CONFIRM_MESSAGE, message));
+    }
+
     public void mode(String graph, GraphMode mode) {
-        String path = String.join("/", this.path(), graph);
         // NOTE: Must provide id for PUT. If use "graph/mode", "/" will
         // be encoded to "%2F". So use "mode" here although inaccurate.
-        this.client.put(path, "mode", mode);
+        this.client.put(joinPath(this.path(), graph, MODE), null, mode);
     }
 
     public GraphMode mode(String graph) {
-        String path = String.join("/", this.path(), graph);
-        RestResult result =  this.client.get(path, "mode");
+        RestResult result = this.client.get(joinPath(this.path(), graph), MODE);
         @SuppressWarnings("unchecked")
         Map<String, String> mode = result.readObject(Map.class);
-        String value = mode.get("mode");
+        String value = mode.get(MODE);
         if (value == null) {
             throw new InvalidResponseException(
                       "Invalid response, expect 'mode' in response");
@@ -81,20 +119,20 @@ public GraphMode mode(String graph) {
 
     public void readMode(String graph, GraphReadMode readMode) {
         this.client.checkApiVersion("0.59", "graph read mode");
-        String path = String.join("/", this.path(), graph);
         // NOTE: Must provide id for PUT. If use "graph/graph_read_mode", "/"
         // will be encoded to "%2F". So use "graph_read_mode" here although
         // inaccurate.
-        this.client.put(path, "graph_read_mode", readMode);
+        this.client.put(joinPath(this.path(), graph, GRAPH_READ_MODE),
+                        null, readMode);
     }
 
     public GraphReadMode readMode(String graph) {
         this.client.checkApiVersion("0.59", "graph read mode");
-        String path = String.join("/", this.path(), graph);
-        RestResult result =  this.client.get(path, "graph_read_mode");
+        RestResult result = this.client.get(joinPath(this.path(), graph),
+                                            GRAPH_READ_MODE);
         @SuppressWarnings("unchecked")
         Map<String, String> readMode = result.readObject(Map.class);
-        String value = readMode.get("graph_read_mode");
+        String value = readMode.get(GRAPH_READ_MODE);
         if (value == null) {
             throw new InvalidResponseException(
                       "Invalid response, expect 'graph_read_mode' in response");
@@ -107,9 +145,11 @@ public GraphReadMode readMode(String graph) {
         }
     }
 
-    public void clear(String graph, String message) {
-        String path = String.join("/", this.path(), graph, "clear");
-        this.client.delete(path,
-                           ImmutableMap.of("confirm_message", message));
+    private static String joinPath(String path, String graph) {
+        return String.join(DELIMITER, path, graph);
+    }
+
+    private static String joinPath(String path, String graph, String action) {
+        return String.join(DELIMITER, path, graph, action);
     }
 }