Merge pull request ceph#59608 from VallariAg/wip-nvmeof-teuthology-v7-upgrade

VallariAg · web-flow · commit 8e926a0e4500 · 2025-09-18T16:27:21.000+05:30
qa/suites/nvmeof: add upgrade sub-suite
diff --git a/qa/suites/nvmeof/upgrade/% b/qa/suites/nvmeof/upgrade/%
diff --git a/qa/suites/nvmeof/upgrade/.qa b/qa/suites/nvmeof/upgrade/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/nvmeof/upgrade/0-clusters/.qa b/qa/suites/nvmeof/upgrade/0-clusters/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/nvmeof/upgrade/0-clusters/4-gateways-1-initiator.yaml b/qa/suites/nvmeof/upgrade/0-clusters/4-gateways-1-initiator.yaml
@@ -0,0 +1,32 @@
+roles:
+- - client.0 # initiator
+- - host.a
+  - mon.a
+  - mgr.x
+  - osd.0
+  - osd.1
+  - client.1
+  - ceph.nvmeof.nvmeof.a
+- - host.b
+  - mon.b
+  - mgr.y
+  - osd.2
+  - osd.3
+  - osd.4
+  - client.2
+  - ceph.nvmeof.nvmeof.b
+- - host.c
+  - mon.c
+  - osd.5
+  - osd.6
+  - osd.7
+  - client.3
+  - ceph.nvmeof.nvmeof.c
+- - host.d
+  - client.4
+  - ceph.nvmeof.nvmeof.d
+
+tasks:
+- install:
+    extra_packages:
+        - nvme-cli
diff --git a/qa/suites/nvmeof/upgrade/1-start-distro/.qa b/qa/suites/nvmeof/upgrade/1-start-distro/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/nvmeof/upgrade/1-start-distro/1-start-centos_9.stream-squid.yaml b/qa/suites/nvmeof/upgrade/1-start-distro/1-start-centos_9.stream-squid.yaml
@@ -0,0 +1,19 @@
+os_type: centos
+os_version: "9.stream"
+
+tasks:
+- cephadm:
+    image: quay.ceph.io/ceph-ci/ceph:squid-nvmeof
+    compiled_cephadm_branch: squid-nvmeof
+
+- nvmeof:
+    installer: host.a
+    rbd:
+      pool_name: mypool
+      image_name_prefix: myimage
+    gateway_config:
+      subsystems_count: 3
+      namespaces_count: 20
+
+- cephadm.wait_for_service:
+    service: nvmeof.mypool.mygroup0
diff --git a/qa/suites/nvmeof/upgrade/2-setup_subsystem.yaml b/qa/suites/nvmeof/upgrade/2-setup_subsystem.yaml
@@ -0,0 +1,10 @@
+tasks:
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.0:
+        - nvmeof/setup_subsystem.sh
+        - nvmeof/basic_tests.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
diff --git a/qa/suites/nvmeof/upgrade/3-upgrade/.qa b/qa/suites/nvmeof/upgrade/3-upgrade/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/nvmeof/upgrade/3-upgrade/simple.yaml b/qa/suites/nvmeof/upgrade/3-upgrade/simple.yaml
@@ -0,0 +1,21 @@
+overrides:
+  ceph:
+    log-ignorelist:
+      - CEPHADM_STRAY_DAEMON
+      - CEPHADM_FAILED_DAEMON
+      - CEPHADM_AGENT_DOWN
+    log-only-match:
+      - CEPHADM_
+
+tasks:
+- cephadm.shell:
+    env: [sha1]
+    mon.a:
+      # print status before upgrade
+      - ceph health detail
+      - ceph orch ps
+      - ceph -s
+      # upgrade
+      - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force
+      - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force
+      - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1
diff --git a/qa/suites/nvmeof/upgrade/4-wait.yaml b/qa/suites/nvmeof/upgrade/4-wait.yaml
@@ -0,0 +1,15 @@
+tasks:
+- cephadm.shell:
+    env: [sha1]
+    mon.a:
+      - while ceph orch upgrade status | jq '.in_progress' | grep true && ! ceph orch upgrade status | jq '.message' | grep Error ; do ceph orch ps ; ceph versions ; ceph orch upgrade status ; ceph health detail ; sleep 30 ; done
+      - ceph config get mgr mgr/cephadm/container_image_nvmeof
+      - sleep 60
+      # print status after upgrade
+      - ceph orch ps
+      - ceph orch ps | grep "nvmeof" || exit 1
+      - ceph orch ps | grep "nvmeof" | grep "running" || exit 1
+      - ceph versions
+      - ceph orch upgrade status
+      - ceph health detail
+      - ceph versions | jq -e '.overall | length == 1'
diff --git a/qa/suites/nvmeof/upgrade/5-workloads/.qa b/qa/suites/nvmeof/upgrade/5-workloads/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/nvmeof/upgrade/5-workloads/nvmeof-thrasher.yaml b/qa/suites/nvmeof/upgrade/5-workloads/nvmeof-thrasher.yaml
@@ -0,0 +1,23 @@
+tasks:
+- workunit:
+    no_coverage_and_limits: true
+    clients:
+      client.0:
+        - nvmeof/basic_tests.sh
+    env:
+      RBD_POOL: mypool
+      RBD_IMAGE_PREFIX: myimage
+
+- nvmeof.thrash:
+    checker_host: 'client.0'
+
+- workunit:
+    no_coverage_and_limits: true
+    timeout: 30m
+    clients:
+      client.0:
+        - nvmeof/fio_test.sh --rbd_iostat
+    env:
+      RBD_POOL: mypool
+      IOSTAT_INTERVAL: '10'
+      RUNTIME: '600'
diff --git a/qa/tasks/nvmeof.py b/qa/tasks/nvmeof.py
@@ -61,6 +61,7 @@ def _set_defaults(self):
 
         gateway_config = self.config.get('gateway_config', {})
         self.cli_image = gateway_config.get('cli_image', 'quay.io/ceph/nvmeof-cli:latest')
+        self.enable_groups = bool(gateway_config.get('enable_groups', True))
         self.groups_count = gateway_config.get('groups_count', 1)
         self.groups_prefix = gateway_config.get('groups_prefix', 'mygroup') 
         self.nqn_prefix = gateway_config.get('subsystem_nqn_prefix', 'nqn.2016-06.io.spdk:cnode')
@@ -114,16 +115,22 @@ def deploy_nvmeof(self):
                 'rbd', 'pool', 'init', poolname
             ])
 
-            group_to_nodes = defaultdict(list)
-            for index, node in enumerate(nodes):
-                group_name = self.groups_prefix + str(index % int(self.groups_count))
-                group_to_nodes[group_name] += [node]
-            for group_name in group_to_nodes:
-                gp_nodes = group_to_nodes[group_name]
-                log.info(f'[nvmeof]: ceph orch apply nvmeof {poolname} {group_name}')
+            if self.enable_groups:
+                group_to_nodes = defaultdict(list)
+                for index, node in enumerate(nodes):
+                    group_name = self.groups_prefix + str(index % int(self.groups_count))
+                    group_to_nodes[group_name] += [node]
+                for group_name in group_to_nodes:
+                    gp_nodes = group_to_nodes[group_name]
+                    log.info(f'[nvmeof]: ceph orch apply nvmeof {poolname} {group_name}')
+                    _shell(self.ctx, self.cluster_name, self.remote, [
+                        'ceph', 'orch', 'apply', 'nvmeof', poolname, group_name,
+                        '--placement', ';'.join(gp_nodes)
+                    ])
+            else:
                 _shell(self.ctx, self.cluster_name, self.remote, [
-                    'ceph', 'orch', 'apply', 'nvmeof', poolname, group_name,
-                    '--placement', ';'.join(gp_nodes)
+                        'ceph', 'orch', 'apply', 'nvmeof', poolname,
+                        '--placement', ';'.join(nodes)
                 ])
 
             total_images = int(self.namespaces_count) * int(self.subsystems_count)
@@ -287,7 +294,7 @@ class NvmeofThrasher(Thrasher, Greenlet):
     - workunit:
         clients:
             client.3:
-            - rbd/nvmeof_fio_test.sh --rbd_iostat
+            - nvmeof/fio_test.sh --rbd_iostat
         env:
             RBD_POOL: mypool
             IOSTAT_INTERVAL: '10'
