Add support for GCP tests

- Implement GCPDeployer/GCPDestroyer classes for infrastructure
deployment/teardown
- Add playbooks to configure server/client instances
- Edit install.py to upload couchbase.rpm to client instances too when
--remote-copy is passed (for tools tests, the client needs this too)
- Provide ability to use internal/private IPs when setting up couchbase
cluster, since GCP instances cannot be renamed to their external/public
IP addresses

Change-Id: I0f8b96a807d26eee9568254bc041741cd149c6c9
Reviewed-on: https://review.couchbase.org/c/perfrunner/+/169032
Tested-by: Build Bot <[email protected]>
Reviewed-by: Korrigan Clark <[email protected]>

Author: d-nagy

cloud/infrastructure/gcp_4s_1c_large_tools.spec

@@ -0,0 +1,56 @@
+[infrastructure]
+provider = gcp
+type = gce
+
+[clusters]
+couchbase1 =
+        gce.gce_cluster_1.gce_node_group_1.1:kv
+        gce.gce_cluster_1.gce_node_group_1.2:kv
+        gce.gce_cluster_1.gce_node_group_1.3:kv
+        gce.gce_cluster_1.gce_node_group_1.4:kv
+
+[clients]
+workers1 =
+        gce.gce_cluster_1.gce_node_group_2.1
+
+[utilities]
+brokers1 = gce.gce_cluster_1.gce_node_group_3.1
+
+[gce]
+clusters = gce_cluster_1
+
+[gce_cluster_1]
+node_groups = gce_node_group_1,gce_node_group_2,gce_node_group_3
+storage_class = pd-extreme
+
+[gce_node_group_1]
+instance_type = n2d-standard-16
+instance_capacity = 4
+volume_size = 1000
+volume_type = pd-extreme
+iops = 20000
+
+[gce_node_group_2]
+instance_type = n2-standard-32
+instance_capacity = 1
+volume_size = 1000
+volume_type = pd-extreme
+iops = 20000
+
+[gce_node_group_3]
+instance_type = n2-standard-32
+instance_capacity = 1
+
+[storage]
+data = /data
+backup = gs://perftest-gcp-backup
+
+[credentials]
+rest = Administrator:password
+ssh = root:couchbase
+
+[parameters]
+OS = CentOS 7
+CPU = 16vCPU
+Memory = 64GB
+Disk = pd-extreme 1TB 20000 IOPS

perfrunner/helpers/cluster.py

@@ -63,6 +63,13 @@ def set_analytics_path(self):
     def rename(self):
         if self.dynamic_infra:
             return
+        elif self.cluster_spec.cloud_infrastructure:
+            if self.cluster_spec.cloud_provider == 'gcp':
+                clusters_private = dict(self.cluster_spec.clusters_private)
+                for cluster_name, public_ips in self.cluster_spec.clusters:
+                    private_ips = clusters_private.get(cluster_name, [])
+                    for host, new_host in zip(public_ips, private_ips):
+                        self.rest.rename(host, new_host)
         else:
             for server in self.cluster_spec.servers:
                 self.rest.rename(server)
@@ -236,15 +243,23 @@ def set_services(self):
     def add_nodes(self):
         if self.dynamic_infra:
             return
-        for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes):
+
+        using_gcp = self.cluster_spec.cloud_provider == 'gcp'
+
+        for (cluster, servers), initial_nodes in zip(self.cluster_spec.clusters,
+                                                     self.initial_nodes):
 
             if initial_nodes < 2:  # Single-node cluster
                 continue
 
+            if using_gcp:
+                private_ips = dict(self.cluster_spec.clusters_private)[cluster]
+
             master = servers[0]
-            for node in servers[1:initial_nodes]:
+            for i, node in enumerate(servers[1:initial_nodes]):
                 roles = self.cluster_spec.roles[node]
-                self.rest.add_node(master, node, roles)
+                new_host = private_ips[1:initial_nodes][i] if using_gcp else node
+                self.rest.add_node(master, new_host, roles)
 
     def rebalance(self):
         if self.dynamic_infra:

perfrunner/helpers/rest.py

@@ -128,11 +128,14 @@ def set_auth(self, host: str):
         }
         self.post(url=api, data=data)
 
-    def rename(self, host: str):
-        logger.info('Changing server name: {}'.format(host))
+    def rename(self, host: str, new_host: str = None):
+        if not new_host:
+            new_host = host
+
+        logger.info('Changing server name: {} -> {}'.format(host, new_host))
 
         api = 'http://{}:8091/node/controller/rename'.format(host)
-        data = {'hostname': host}
+        data = {'hostname': new_host}
 
         self.post(url=api, data=data)
 

perfrunner/settings.py

@@ -72,6 +72,10 @@ def cloud_infrastructure(self):
         else:
             return False
 
+    @property
+    def cloud_provider(self):
+        return self.config.get('infrastructure', 'provider', fallback='')
+
     @property
     def kubernetes_infrastructure(self):
         if self.cloud_infrastructure:
@@ -152,6 +156,12 @@ def servers(self) -> List[str]:
                 servers.append(server)
         return servers
 
+    @property
+    def clusters_private(self) -> Iterator:
+        if self.config.has_section('private_ips'):
+            for cluster_name, private_ips in self.config.items('private_ips'):
+                yield cluster_name, private_ips.split()
+
     def servers_by_role(self, role: str) -> List[str]:
         has_service = []
         for _, servers in self.config.items('clusters'):