create collections via bulk api, off by default

Change-Id: I042b6d0fc43c9f84cd67156faf7ea916c1cb8baf
Reviewed-on: http://review.couchbase.org/c/perfrunner/+/149225
Tested-by: Build Bot <[email protected]>
Reviewed-by: Bo-Chun Wang <[email protected]>

Author: korry8911

perfrunner/helpers/cluster.py

@@ -298,23 +298,34 @@ def create_collections(self):
         collection_map = self.test_config.collection.collection_map
         for master in self.cluster_spec.masters:
             if collection_map is not None:
-                for bucket in collection_map.keys():
-                    delete_default = True
-                    for scope in collection_map[bucket]:
-                        if scope == '_default':
+                if self.test_config.collection.use_bulk_api:
+                    for bucket in collection_map.keys():
+                        create_scopes = []
+                        for scope in collection_map[bucket]:
+                            scope_collections = []
                             for collection in collection_map[bucket][scope]:
-                                if collection == "_default":
-                                    delete_default = False
-                    if delete_default:
-                        self.rest.delete_collection(master, bucket, '_default', '_default')
-
-                for bucket in collection_map.keys():
-                    for scope in collection_map[bucket]:
-                        if scope != '_default':
-                            self.rest.create_scope(master, bucket, scope)
-                        for collection in collection_map[bucket][scope]:
-                            if collection != '_default':
-                                self.rest.create_collection(master, bucket, scope, collection)
+                                scope_collections.append({"name": collection})
+                            create_scopes.append({"name": scope, "collections": scope_collections})
+                        print(str({"scopes": create_scopes}))
+                        self.rest.set_collection_map(master, bucket, {"scopes": create_scopes})
+                else:
+                    for bucket in collection_map.keys():
+                        delete_default = True
+                        for scope in collection_map[bucket]:
+                            if scope == '_default':
+                                for collection in collection_map[bucket][scope]:
+                                    if collection == "_default":
+                                        delete_default = False
+                        if delete_default:
+                            self.rest.delete_collection(master, bucket, '_default', '_default')
+
+                    for bucket in collection_map.keys():
+                        for scope in collection_map[bucket]:
+                            if scope != '_default':
+                                self.rest.create_scope(master, bucket, scope)
+                            for collection in collection_map[bucket][scope]:
+                                if collection != '_default':
+                                    self.rest.create_collection(master, bucket, scope, collection)
 
     def create_eventing_buckets(self):
         if not self.test_config.cluster.eventing_bucket_mem_quota:

perfrunner/helpers/rest.py

@@ -1026,6 +1026,12 @@ def delete_collection(self, host, bucket, scope, collection):
             .format(host, bucket, scope, collection)
         self.delete(url=api)
 
+    def set_collection_map(self, host, bucket, collection_map):
+        logger.info("Setting collection map on {} via bulk api".format(bucket))
+        api = 'http://{}:8091/pools/default/buckets/{}/collections' \
+            .format(host, bucket)
+        self.put(url=api, data=json.dumps(collection_map))
+
 
 class KubernetesRestHelper(RestBase):
 

perfrunner/settings.py

@@ -453,10 +453,12 @@ class CollectionSettings:
 
     CONFIG = None
     COLLECTION_MAP = None
+    USE_BULK_API = 0
 
     def __init__(self, options: dict):
         self.config = options.get('config', self.CONFIG)
         self.collection_map = self.COLLECTION_MAP
+        self.use_bulk_api = int(options.get('use_bulk_api', self.USE_BULK_API))
         if self.config is not None:
             with open(self.config) as f:
                 self.collection_map = json.load(f)