python 3.9 upgrade

Change-Id: I13b9448458b34d122c1b5bc5809adafbbf61d36a
Reviewed-on: https://review.couchbase.org/c/perfrunner/+/163891
Reviewed-by: Bo-Chun Wang <[email protected]>
Tested-by: Korrigan Clark <[email protected]>

Author: korry8911

Makefile

@@ -3,15 +3,16 @@ SHELL := /bin/bash
 PATH := ${GOPATH}/bin:$(PATH)
 
 ENV := env
-PYTHON := python3.6
+PYTHON := python3.9
 PYTHON_PROJECTS := cbagent perfdaily perfrunner scripts spring
 .PHONY: docker
 
 all:
 	export PYENV_ROOT="$$HOME/.pyenv" && \
 	export PATH="$$PYENV_ROOT/bin:$$PATH" && \
-	eval "$$(pyenv init -)" && \
-	pyenv local 3.6.12 && \
+	eval "$$(pyenv init --path)" && \
+	pyenv install 3.9.7 -s && \
+	pyenv local 3.9.7 && \
 	virtualenv --quiet --python ${PYTHON} ${ENV}
 	${ENV}/bin/pip install --upgrade --quiet pip wheel
 	${ENV}/bin/pip install --quiet --no-warn-script-location -r requirements.txt
@@ -70,7 +71,7 @@ docker:
 	docker build -t docker.io/perflab/perfrunner docker
 
 docker-cloud-worker:
-	pyenv local 3.6.12 && \
+	pyenv local 3.9.7 && \
 	virtualenv --quiet --python ${PYTHON} ${ENV}
 	${ENV}/bin/pip install --upgrade --quiet pip wheel
 	${ENV}/bin/pip install --quiet --no-warn-script-location -r requirements.txt

cbagent/collectors/eventing_stats.py

@@ -145,20 +145,20 @@ def _get_pid_stats(self, node_pids):
                 pid_list_top = []
                 pid_list_top_str = ""
                 grep_text_top = []
-                grep_text_top_str = "^\s*"
+                grep_text_top_str = "^\\s*"
                 counter = 0
                 # run top command in batches of 20 pids as per top command limitation
                 for pid in pids.values():
-                    grep_text_ps += '^[[:space:]]*{}\|'.format(pid)
+                    grep_text_ps += '^[[:space:]]*{}\\|'.format(pid)
                     pid_list_top_str += '{},'.format(pid)
-                    grep_text_top_str += '{}\|'.format(pid)
+                    grep_text_top_str += '{}\\|'.format(pid)
                     counter += 1
                     if counter == 20:
                         counter = 0
                         pid_list_top.append(pid_list_top_str[:-1])
                         grep_text_top.append(grep_text_top_str[:-2])
                         pid_list_top_str = ""
-                        grep_text_top_str = "^\s*"
+                        grep_text_top_str = "^\\s*"
                 grep_text_ps = grep_text_ps[:-2]
                 with settings(host_string=node):
                     rss_used = run(self.PS_CMD.format(grep_text_ps))

cbagent/collectors/libstats/pcstat.py

@@ -7,7 +7,7 @@ class PCStat(RemoteStats):
 
     def get_pcstat(self, partition: str) -> float:
         stdout = self.run(
-            'find {} -regex .*[0-9]+\.couch\.[0-9]+ \
+            'find {} -regex .*[0-9]+\\.couch\\.[0-9]+ \
             | xargs pcstat -nohdr -terse 2>/dev/null'.format(partition),
         )
         percents = []

cbagent/collectors/libstats/psstats.py

@@ -11,7 +11,7 @@ class PSStats(RemoteStats):
     PS_CMD = "ps -eo pid,rss,vsize,comm | " \
         "grep {} | grep -v grep | sort -n -k 2 | tail -n 1"
 
-    TOP_CMD = "top -b -n2 -d{0} -p {1} | grep '^\s*{1}' | tail -n 1"
+    TOP_CMD = "top -b -n2 -d{0} -p {1} | grep '^\\s*{1}' | tail -n 1"
 
     MAX_TOP_INTERVAL = 10  # seconds
 

cbagent/stores.py

@@ -19,7 +19,7 @@ def build_dbname(cluster: str,
                      index: str = None,
                      collector: str = None) -> str:
         db_name = (collector or "") + cluster + (bucket or "") + (index or "") + (server or "")
-        for char in "[]/\;.,><&*:%=+@!#^()|?^'\"":
+        for char in "[ ] / \\ ; . , > < & * : % = + @ ! # ^ ( ) | ? ^ ' \"":
             db_name = db_name.replace(char, "")
         return db_name
 

perfrunner/celerylocal.py

@@ -1,7 +1,14 @@
+from kombu.serialization import registry
+
 broker_url = 'sqla+sqlite:///perfrunner.db'
 result_backend = 'database'
 database_url = 'sqlite:///results.db'
 task_serializer = 'pickle'
 result_serializer = 'pickle'
-accept_content = {'pickle'}
+accept_content = {'pickle', 'json', 'application/json', 'application/data', 'application/text'}
 task_protocol = 1
+
+registry.enable('json')
+registry.enable('application/json')
+registry.enable('application/data')
+registry.enable('application/text')

perfrunner/celeryremote.py

@@ -1,13 +1,20 @@
+from kombu.serialization import registry
+
 broker_url = 'amqp://couchbase:[email protected]:5672/broker'
 broker_pool_limit = None
 worker_hijack_root_logger = False
-result_backend = "amqp://"
+result_backend = "rpc://"
 result_persistent = False
 result_exchange = "perf_results"
-accept_content = ['pickle']
+accept_content = ['pickle', 'json', 'application/json', 'application/data', 'application/text']
 result_serializer = 'pickle'
 task_serializer = 'pickle'
 task_protocol = 1
 broker_connection_timeout = 60
 broker_connection_retry = True
 broker_connection_max_retries = 100
+
+registry.enable('json')
+registry.enable('application/json')
+registry.enable('application/data')
+registry.enable('application/text')

perfrunner/helpers/cluster.py

@@ -236,16 +236,15 @@ def set_services(self):
     def add_nodes(self):
         if self.dynamic_infra:
             return
-        for (_, servers), initial_nodes \
-                in zip(self.cluster_spec.clusters, self.initial_nodes):
+        for (_, servers), initial_nodes in zip(self.cluster_spec.clusters, self.initial_nodes):
 
-                if initial_nodes < 2:  # Single-node cluster
-                    continue
+            if initial_nodes < 2:  # Single-node cluster
+                continue
 
-                master = servers[0]
-                for node in servers[1:initial_nodes]:
-                    roles = self.cluster_spec.roles[node]
-                    self.rest.add_node(master, node, roles)
+            master = servers[0]
+            for node in servers[1:initial_nodes]:
+                roles = self.cluster_spec.roles[node]
+                self.rest.add_node(master, node, roles)
 
     def rebalance(self):
         if self.dynamic_infra:

perfrunner/helpers/local.py

@@ -866,8 +866,8 @@ def start_celery_worker(queue: str):
     with shell_env(PYTHONOPTIMIZE='1', PYTHONWARNINGS='ignore', C_FORCE_ROOT='1'):
         local('WORKER_TYPE=local '
               'BROKER_URL=sqla+sqlite:///perfrunner.db '
-              'nohup env/bin/celery worker '
-              '-A perfrunner.helpers.worker -l INFO -Q {} -f worker.log &'.format(queue))
+              'nohup env/bin/celery -A perfrunner.helpers.worker worker '
+              '-l INFO -Q {} -f worker.log &'.format(queue))
 
 
 def clone_git_repo(repo: str, branch: str, commit: str = None):

perfrunner/helpers/monitor.py

@@ -322,7 +322,7 @@ def _get_num_items(self, host: str, bucket: str, total: bool = False) -> int:
             return curr_items[-1]
         return 0
 
-    def monitor_num_items(self, host: str, bucket: str, num_items: int, max_retry: int=None):
+    def monitor_num_items(self, host: str, bucket: str, num_items: int, max_retry: int = None):
         logger.info('Checking the number of items in {}'.format(bucket))
         if not max_retry:
             max_retry = self.MAX_RETRY
@@ -1040,7 +1040,7 @@ def _wait_for_replica_count_match(self, host, bucket):
             if time.time() - start_time > self.TIMEOUT:
                 raise Exception('Replica items monitoring got stuck')
 
-    def monitor_num_items(self, host: str, bucket: str, num_items: int, max_retry: int=None):
+    def monitor_num_items(self, host: str, bucket: str, num_items: int, max_retry: int = None):
         logger.info('Checking the number of items in {}'.format(bucket))
         if not max_retry:
             max_retry = self.MAX_RETRY