CBPS-1148: Prepare perfrunner for using Ubuntu 20 for on-prem servers

Change-Id: I75c19917d448a098aa5e574146fc68c1f3ded1a7
Reviewed-on: https://review.couchbase.org/c/perfrunner/+/190697
Tested-by: Build Bot <[email protected]>
Reviewed-by: Salim Salim <[email protected]>
Reviewed-by: Bo-Chun Wang <[email protected]>

Author: d-nagy

cbagent/collectors/libstats/iostat.py

@@ -1,21 +1,11 @@
-from typing import Dict, Union
+from typing import Dict, Optional, Tuple
 
 from cbagent.collectors.libstats.remotestats import RemoteStats, parallel_task
 
 
 class IOStat(RemoteStats):
 
-    METRICS_Centos = (
-        ("rps", "r/s", 1),
-        ("wps", "w/s", 1),
-        ("rbps", "rkB/s", 1024),  # kB -> B
-        ("wbps", "wkB/s", 1024),  # kB -> B
-        ("avgqusz", "avgqu-sz", 1),
-        ("await", "await", 1),
-        ("util", "%util", 1),
-    )
-
-    METRICS_Ubuntu = (
+    METRICS = (
         ("rps", "r/s", 1),
         ("wps", "w/s", 1),
         ("rbps", "rkB/s", 1024),  # kB -> B
@@ -24,7 +14,7 @@ class IOStat(RemoteStats):
         ("util", "%util", 1),
     )
 
-    def get_device_name(self, path: str) -> [Union[None, str], bool]:
+    def get_device_name(self, path: str) -> Tuple[Optional[str], bool]:
         stdout = self.run("df '{}' | head -2 | tail -1".format(path),
                           quiet=True)
         if not stdout.return_code:
@@ -50,33 +40,21 @@ def get_iostat(self, device: str) -> Dict[str, str]:
 
     @parallel_task(server_side=True)
     def get_server_samples(self, partitions: dict) -> dict:
-        return self.get_samples_centos(partitions['server'])
+        return self.get_samples(partitions['server'], self.METRICS)
 
     @parallel_task(server_side=False)
     def get_client_samples(self, partitions: dict) -> dict:
-        return self.get_samples_ubuntu(partitions['client'])
-
-    def get_samples_centos(self, partitions: Dict[str, str]) -> Dict[str, float]:
-        samples = {}
-
-        for purpose, path in partitions.items():
-            device, _ = self.get_device_name(path)
-            if device is not None:
-                stats = self.get_iostat(device)
-                for metric, column, multiplier in self.METRICS_Centos:
-                    key = "{}_{}".format(purpose, metric)
-                    samples[key] = float(stats[column]) * multiplier
-
-        return samples
+        return self.get_samples(partitions['client'], self.METRICS)
 
-    def get_samples_ubuntu(self, partitions: Dict[str, str]) -> Dict[str, float]:
+    def get_samples(self, partitions: Dict[str, str],
+                    metrics: Tuple[Tuple[str, str, int]]) -> Dict[str, float]:
         samples = {}
 
         for purpose, path in partitions.items():
             device, _ = self.get_device_name(path)
             if device is not None:
                 stats = self.get_iostat(device)
-                for metric, column, multiplier in self.METRICS_Ubuntu:
+                for metric, column, multiplier in metrics:
                     key = "{}_{}".format(purpose, metric)
                     samples[key] = float(stats[column]) * multiplier
 

cbagent/collectors/libstats/pcstat.py

@@ -1,3 +1,5 @@
+from typing import Tuple
+
 import numpy
 
 from cbagent.collectors.libstats.remotestats import RemoteStats, parallel_task
@@ -17,7 +19,7 @@ def get_pcstat(self, partition: str) -> float:
             percents.append(float(percent))
         return numpy.average(percents)
 
-    def get_cachestat(self) -> [float, float]:
+    def get_cachestat(self) -> Tuple[float, float]:
         stdout = self.run('cachestat')
         total_hits, hit_ratio = stdout.split()
         return float(total_hits), float(hit_ratio)

perfrunner/helpers/cbmonitor.py

@@ -212,7 +212,7 @@ def add_collectors(self,
 
         if not self.test.dynamic_infra:
             if not self.test.capella_infra:
-                if self.test.remote.os != 'Cygwin':
+                if self.test.remote.PLATFORM != 'cygwin':
                     self.add_collector(PS)
                     self.add_collector(Sysdig)
                     if memory:

perfrunner/helpers/cluster.py

@@ -913,7 +913,7 @@ def throttle_cpu(self):
                 cluster['spec']['servers'] = updated_server_groups
                 self.remote.update_cluster_config(cluster)
         else:
-            if self.remote.os == 'Cygwin':
+            if self.remote.PLATFORM == 'cygwin':
                 return
 
             if self.test_config.cluster.enable_cpu_cores:
@@ -1045,9 +1045,13 @@ def set_min_tls_version(self):
             check_tls_version = self.rest.get_minimum_tls_version(self.master_node)
             logger.info('new tls version: {}'.format(check_tls_version))
 
-    def get_debug_rpm_url(self):
+    def get_debug_package_url(self):
         release, build_number = self.build.split('-')
-        if self.build_tuple > (7, 2, 0, 0):
+        if self.build_tuple > (8, 0, 0, 0):
+            release = 'morpheus'
+        elif self.build_tuple > (7, 6, 0, 0):
+            release = 'trinity'
+        elif (7, 2, 0, 0) < self.build_tuple <= (7, 2, 0, 2228) or self.build_tuple > (7, 5, 0, 0):
             release = 'elixir'
         elif self.build_tuple > (7, 1, 0, 0):
             release = 'neo'
@@ -1057,16 +1061,21 @@ def get_debug_rpm_url(self):
             release = 'mad-hatter'
         elif self.build_tuple < (6, 5, 0, 0):
             release = 'alice'
-        centos_version = self.remote.detect_centos_release()
 
-        rpm_url = 'http://latestbuilds.service.couchbase.com/builds/' \
-                  'latestbuilds/couchbase-server/{}/{}/' \
-                  'couchbase-server-enterprise-debuginfo-{}-centos{}.x86_64.rpm' \
-                  ''.format(release, build_number, self.build, centos_version)
-        return rpm_url
+        if self.remote.distro.upper() in ['UBUNTU', 'DEBIAN']:
+            package_name = 'couchbase-server-enterprise-dbg_{{}}-{{}}{{}}_amd64.deb'
+        else:
+            package_name = 'couchbase-server-enterprise-debuginfo-{{}}-{{}}{{}}.x86_64.rpm'
+
+        package_name = package_name.format(self.build, self.remote.distro,
+                                           self.remote.distro_version)
+        return (
+            'http://latestbuilds.service.couchbase.com/'
+            'builds/latestbuilds/couchbase-server/{}/{}/{}'
+        ).format(release, build_number, package_name)
 
-    def install_cb_debug_rpm(self):
-        self.remote.install_cb_debug_rpm(url=self.get_debug_rpm_url())
+    def install_cb_debug_package(self):
+        self.remote.install_cb_debug_package(url=self.get_debug_package_url())
 
     def enable_developer_preview(self):
         if self.build_tuple > (7, 0, 0, 4698) or self.build_tuple < (1, 0, 0, 0):

perfrunner/helpers/local.py

@@ -551,7 +551,7 @@ def run_kvgen(hostname: str, num_docs: int, prefix: str):
 
 
 def build_ycsb(ycsb_client: str):
-    cmd = 'pyenv local system && bin/ycsb build {}'.format(ycsb_client)
+    cmd = 'pyenv local 2 && bin/ycsb build {}'.format(ycsb_client)
 
     logger.info('Running: {}'.format(cmd))
     with lcd('YCSB'):
@@ -629,7 +629,7 @@ def run_ycsb(host: str,
           '-p couchbase.retryUpper={retry_upper} ' \
           '-p couchbase.retryFactor={retry_factor} '
 
-    cmd = 'pyenv local system && ' + cmd
+    cmd = 'pyenv local 2 && ' + cmd
 
     if durability is None:
         cmd += '-p couchbase.persistTo={persist_to} '

perfrunner/helpers/remote.py

@@ -1,3 +1,5 @@
+from typing import Tuple
+
 from fabric import state
 from fabric.api import run, settings
 
@@ -14,7 +16,7 @@ def __new__(cls, cluster_spec: ClusterSpec, verbose: bool = False):
         if not cluster_spec.ssh_credentials:
             return None
         if cluster_spec.dynamic_infrastructure:
-            return RemoteKubernetes(cluster_spec, 'kubernetes')
+            return RemoteKubernetes(cluster_spec)
 
         state.env.user, state.env.password = cluster_spec.ssh_credentials
         state.output.running = verbose
@@ -23,32 +25,33 @@ def __new__(cls, cluster_spec: ClusterSpec, verbose: bool = False):
         if cluster_spec.capella_infrastructure:
             state.env.use_ssh_config = True
 
-        if cluster_spec.cloud_infrastructure:
-            if cluster_spec.infrastructure_settings.get('os_arch', 'x86_64') == 'arm':
-                os = 'arm'
-            elif cluster_spec.infrastructure_settings.get('os_arch', 'x86_64') == 'al2':
-                os = 'al2'
-            elif cluster_spec.capella_infrastructure:
-                os = 'linux'
-            else:
-                os = cls.detect_server_os(cluster_spec.servers[0], cluster_spec)
-        else:
-            os = cls.detect_server_os(cluster_spec.servers[0], cluster_spec)
-        if os == 'Cygwin':
-            return RemoteWindows(cluster_spec, os)
-        else:
-            return RemoteLinux(cluster_spec, os)
+        os_platform = cls.detect_os_platform(cluster_spec.servers[0], cluster_spec.ssh_credentials)
+
+        if os_platform.startswith('cygwin'):
+            return RemoteWindows(cluster_spec)
+
+        return RemoteLinux(cluster_spec)
 
     @staticmethod
-    def detect_server_os(server: str, cluster_spec: ClusterSpec):
-        state.env.user, state.env.password = cluster_spec.ssh_credentials
-        logger.info('Detecting OS on server {}'.format(server))
-        with settings(host_string=server):
-            os = run('python -c "import platform; print platform.dist()[0]"')
-        if os:
-            return os
-        else:
-            return 'Cygwin'
+    def detect_os_platform(host: str, credentials: Tuple[str, str]) -> str:
+        state.env.user, state.env.password = credentials
+        logger.info('Detecting OS platform on host {} using Python'.format(host))
+
+        with settings(host_string=host):
+            platform = run('python3 -c "import platform; print(platform.system())"', warn_only=True)
+            if platform.return_code != 0:
+                platform = run('python -c "import platform; print platform.system()"',
+                               warn_only=True)
+
+        if not platform:
+            logger.warn('Could not determine OS platform on host {}. Assuming "linux" by default.'
+                        .format(host))
+            platform = 'linux'
+
+        platform = platform.lower()
+        logger.info('Detected OS platform is {}'.format(platform))
+
+        return platform
 
     @staticmethod
     def detect_client_os(server: str, cluster_spec: ClusterSpec):

perfrunner/remote/__init__.py

@@ -17,8 +17,7 @@ class Remote:
 
     CLIENT_PROCESSES = 'celery', 'cbc-pillowfight', 'memcached', 'cblite'
 
-    def __init__(self, cluster_spec, os):
-        self.os = os
+    def __init__(self, cluster_spec):
         self.cluster_spec = cluster_spec
 
     @staticmethod
@@ -85,7 +84,7 @@ def clone_git_repo(self, repo: str, branch: str, worker_home: str, commit: str =
 
     @all_clients
     def build_ycsb(self, worker_home: str, ycsb_client: str):
-        cmd = 'pyenv local system && bin/ycsb build {}'.format(ycsb_client)
+        cmd = 'pyenv local 2 && bin/ycsb build {}'.format(ycsb_client)
 
         logger.info('Running: {}'.format(cmd))
         with cd(worker_home), cd('perfrunner'), cd('YCSB'):

perfrunner/remote/kubernetes.py

@@ -16,8 +16,10 @@
 
 class RemoteKubernetes(Remote):
 
-    def __init__(self, cluster_spec: ClusterSpec, os):
-        super().__init__(cluster_spec, os)
+    PLATFORM = 'kubernetes'
+
+    def __init__(self, cluster_spec: ClusterSpec):
+        super().__init__(cluster_spec)
         self.kube_config_path = "cloud/infrastructure/generated/kube_configs/k8s_cluster_1"
         self.base_path = 'cloud/operator'
         self.cluster_file = 'couchbase-cluster.yaml'
@@ -568,7 +570,7 @@ def build_ycsb(self, worker_home: str, ycsb_client: str):
             line = line.decode("utf-8")
             if "worker" in line:
                 worker_name = line.split()[0]
-                cmd = 'pyenv local system && bin/ycsb build {}'.format(ycsb_client)
+                cmd = 'pyenv local 2 && bin/ycsb build {}'.format(ycsb_client)
 
                 logger.info('Running: {}'.format(cmd))
                 self.kubectl_exec(worker_name, 'cd YCSB; {}'.format(cmd))

perfrunner/remote/linux.py

@@ -1,7 +1,7 @@
 import os
 import time
 from collections import defaultdict
-from typing import Dict, List, Optional
+from typing import Dict, List, Optional, Tuple
 from urllib.parse import urlparse
 
 from fabric.api import cd, get, put, quiet, run, settings
@@ -25,6 +25,8 @@
 
 class RemoteLinux(Remote):
 
+    PLATFORM = 'linux'
+
     CB_DIR = '/opt/couchbase'
 
     PROCESSES = ('beam.smp', 'memcached', 'epmd', 'cbq-engine', 'indexer',
@@ -37,28 +39,27 @@ class RemoteLinux(Remote):
 
     LINUX_PERF_DELAY = 30
 
+    def __init__(self, cluster_spec: ClusterSpec):
+        super().__init__(cluster_spec)
+        self.distro, self.distro_version = self.detect_distro()
+
     @property
     def package(self):
-        if self.os.upper() in ('UBUNTU', 'DEBIAN'):
+        if self.distro.upper() in ('UBUNTU', 'DEBIAN'):
             return 'deb'
         else:
             return 'rpm'
 
     @master_server
-    def detect_centos_release(self) -> str:
-        """Detect CentOS release (e.g., 6 or 7).
-
-        Possible values:
-        - CentOS release 6.x (Final)
-        - CentOS Linux release 7.2.1511 (Core)
-        """
-        return run('cat /etc/redhat-release').split()[-2][0]
-
-    @master_server
-    def detect_ubuntu_release(self):
-        return run('lsb_release -sr').strip()
-
-    def run_cbindex_command(self, options,  worker_home='/tmp/perfrunner'):
+    def detect_distro(self) -> Tuple[str, str]:
+        logger.info('Detecting Linux distribution on master node')
+        cmd = 'grep ^{}= /etc/os-release | cut -d= -f2 | tr -d \'"\''
+        distro_id = run(cmd.format('ID'))
+        distro_version = run(cmd.format('VERSION_ID'))
+        logger.info('Detected Linux distribution: {} {}'.format(distro_id, distro_version))
+        return distro_id, distro_version
+
+    def run_cbindex_command(self, options, worker_home='/tmp/perfrunner'):
         cmd = "/opt/couchbase/bin/cbindex {options}".format(options=options)
         logger.info('Running: {}'.format(cmd))
         run(cmd, shell_escape=False, pty=False)
@@ -341,7 +342,7 @@ def install_couchbase(self, url: str):
         logger.info('Installing Couchbase Server')
         if self.package == 'deb':
             run('yes | apt-get install gdebi')
-            run('yes | apt install -y ./tmp/{}'.format(filename))
+            run('yes | apt install -y /tmp/{}'.format(filename))
         else:
             run('yes | yum localinstall -y /tmp/{}'.format(filename))
 
@@ -1030,9 +1031,14 @@ def cbbackupmgr_restore(self, master_node: str, cluster_spec: ClusterSpec, threa
             run(cmd)
 
     @all_servers
-    def install_cb_debug_rpm(self, url):
-        logger.info('Installing Couchbase Debug rpm on all servers')
-        run('rpm -iv {}'.format(url), quiet=True)
+    def install_cb_debug_package(self, url):
+        logger.info('Installing Couchbase Debug package on all servers')
+        if url.endswith('deb'):
+            cmd = 'dpkg -i {}'.format(url)
+        else:
+            cmd = 'rpm -iv {}'.format(url)
+
+        run(cmd, quiet=True)
 
     @all_servers
     def generate_linux_perf_script(self):

perfrunner/remote/windows.py

@@ -11,6 +11,8 @@
 
 class RemoteWindows(Remote):
 
+    PLATFORM = 'cygwin'
+
     CB_DIR = '/cygdrive/c/Program\\ Files/Couchbase/Server'
 
     VERSION_FILE = '/cygdrive/c/Program Files/Couchbase/Server/VERSION.txt'