qa/suites/nvmeof/basic: add nvmeof_scalability test

Add test to upscale/downscale nvmeof
gateways.

Signed-off-by: Vallari Agrawal <[email protected]>

Author: VallariAg

qa/suites/nvmeof/basic/base/install.yaml

@@ -3,8 +3,7 @@ tasks:
 - install:
     extra_packages:
         - nvme-cli
-- cephadm: 
-    watchdog_setup:
+- cephadm:
 - cephadm.shell:
     host.a:
     # get state before nvmeof deployment

qa/suites/nvmeof/basic/workloads/nvmeof_namespaces.yaml

@@ -35,6 +35,5 @@ tasks:
     env:
       RBD_POOL: mypool
       IOSTAT_INTERVAL: '10'
-      RUNTIME: '600'
+      RUNTIME: '120'
       NEW_NAMESPACES_COUNT: '5'
-

qa/suites/nvmeof/basic/workloads/nvmeof_scalability.yaml

@@ -0,0 +1,39 @@
+tasks:
+- nvmeof:
+    client: client.0
+    gw_image: quay.io/ceph/nvmeof:1.2 # "default" is the image cephadm defaults to; change to test specific nvmeof images, example "latest"
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 3
+      namespaces_count: 20
+      cli_image: quay.io/ceph/nvmeof-cli:1.2
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool      
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.2:
+        - rbd/nvmeof_setup_subsystem.sh
+        - rbd/nvmeof_basic_tests.sh
+        - rbd/nvmeof_fio_test.sh --rbd_iostat
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
+      IOSTAT_INTERVAL: '10'
+      RUNTIME: '60'
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.2:
+        - rbd/nvmeof_scalability_test.sh nvmeof.a
+        - rbd/nvmeof_scalability_test.sh nvmeof.b 
+    env:
+      SCALING_DELAYS: '50'
+

qa/tasks/nvmeof.py

@@ -17,7 +17,7 @@
 log = logging.getLogger(__name__)
 
 conf_file = '/etc/ceph/nvmeof.env'
-
+gw_yaml_file = '/etc/ceph/nvmeof-gw.yaml'
 
 class Nvmeof(Task):
     """
@@ -139,6 +139,9 @@ def deploy_nvmeof(self):
 
         for role, i in daemons.items():
             remote, id_ = i
+            _shell(self.ctx, self.cluster_name, remote, [
+                'ceph', 'orch', 'ls', 'nvmeof', '--export', run.Raw('>'), gw_yaml_file
+            ])
             self.ctx.daemons.register_daemon(
                 remote, 'nvmeof', id_,
                 cluster=self.cluster_name,

qa/workunits/rbd/nvmeof_scalability_test.sh

@@ -0,0 +1,39 @@
+#!/bin/bash -xe
+
+
+GATEWAYS=$1 # exmaple "nvmeof.a,nvmeof.b"
+DELAY="${SCALING_DELAYS:-50}"
+
+if [ -z "$GATEWAYS" ]; then
+    echo "At least one gateway needs to be defined for scalability test"
+    exit 1
+fi
+
+pip3 install yq
+
+status_checks() {
+    ceph nvme-gw show mypool ''
+    ceph orch ls
+    ceph orch ps 
+    ceph -s
+}
+
+
+echo "[nvmeof.scale] Setting up config to remove gateways ${GATEWAYS}"
+ceph orch ls nvmeof --export > /tmp/nvmeof-gw.yaml
+cat /tmp/nvmeof-gw.yaml
+yq "del(.placement.hosts[] | select(. | test(\".*($(echo $GATEWAYS | sed 's/,/|/g'))\")))" /tmp/nvmeof-gw.yaml > /tmp/nvmeof-gw-new.yaml
+cat /tmp/nvmeof-gw-new.yaml
+
+echo "[nvmeof.scale] Starting scale testing by removing ${GATEWAYS}"
+status_checks
+ceph orch rm nvmeof.mypool && sleep 20 # temp workaround
+ceph orch apply -i /tmp/nvmeof-gw-new.yaml # downscale
+sleep $DELAY
+status_checks
+ceph orch rm nvmeof.mypool && sleep 20 # temp workaround
+ceph orch apply -i /tmp/nvmeof-gw.yaml #upscale
+sleep $DELAY
+status_checks
+
+echo "[nvmeof.scale] Scale testing passed for ${GATEWAYS}"