qa/suites/crimson-rados:Seastore (recovery) thrash tests

Matan-B · Matan-B · commit 1800172eb5fd · 2025-05-18T08:30:16.000Z
Seastore is currently only being tested with thrash_simpe without recovery. This commit adds recovery thrash tests with radosbench only for now. Other workloads, mainly `ceph_test_rados` (rados) are not yet supported. See: https://tracker.ceph.com/issues/71237 Signed-off-by: Matan Breizman <mbreizma@redhat.com> (cherry picked from commit 88f2269)
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/% b/qa/suites/crimson-rados/thrash_seastore_radosbench/%
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/0-size-min-size-overrides/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/0-size-min-size-overrides/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/0-size-min-size-overrides/2-size-2-min-size.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/0-size-min-size-overrides/2-size-2-min-size.yaml.disabled
@@ -0,0 +1 @@
+.qa/overrides/2-size-2-min-size.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/0-size-min-size-overrides/3-size-2-min-size.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/0-size-min-size-overrides/3-size-2-min-size.yaml
@@ -0,0 +1 @@
+.qa/overrides/3-size-2-min-size.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/1-pg-log-overrides/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/1-pg-log-overrides/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/1-pg-log-overrides/normal_pg_log.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/1-pg-log-overrides/normal_pg_log.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/1-pg-log-overrides/short_pg_log.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/1-pg-log-overrides/short_pg_log.yaml
@@ -0,0 +1 @@
+.qa/overrides/short_pg_log.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/$ b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/$
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/default.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/default.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-active-recovery.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-active-recovery.yaml.disabled
@@ -0,0 +1 @@
+.qa/overrides/more-active-recovery.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-async-partial-recovery.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-async-partial-recovery.yaml.disabled
@@ -0,0 +1,6 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_async_recovery_min_cost: 1
+        osd_object_clean_region_max_num_intervals: 1000
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-async-recovery.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-async-recovery.yaml.disabled
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_async_recovery_min_cost: 1
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-partial-recovery.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/2-recovery-overrides/more-partial-recovery.yaml.disabled
@@ -0,0 +1,5 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        osd_object_clean_region_max_num_intervals: 1000
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/+ b/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/+
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/fixed-2.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/fixed-2.yaml
@@ -0,0 +1,24 @@
+roles:
+- [mon.a, osd.0, osd.1, client.0, node-exporter.a]
+- [mgr.x, osd.2, osd.3, client.1, prometheus.a, node-exporter.b]
+overrides:
+  ceph:
+    conf:
+      osd:
+        osd shutdown pgref assert: true
+        crimson alien thread cpu cores: 6-7
+      osd.0:
+        crimson seastar cpu cores: 0-2
+      osd.1:
+        crimson seastar cpu cores: 3-5
+      osd.2:
+        crimson seastar cpu cores: 0-2
+      osd.3:
+        crimson seastar cpu cores: 3-5
+      global:
+        ms cluster mode: crc
+        ms service mode: crc
+        ms client mode: crc
+        ms mon service mode: crc
+        ms mon cluster mode: crc
+        ms mon client mode: crc
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/openstack.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/clusters/openstack.yaml.disabled
@@ -0,0 +1,4 @@
+openstack:
+  - volumes: # attached to each instance
+      count: 4
+      size: 10 # GB
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/crimson-supported-all-distro b/qa/suites/crimson-rados/thrash_seastore_radosbench/crimson-supported-all-distro
@@ -0,0 +1 @@
+.qa/distros/crimson-supported-all-distro/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/crimson_qa_overrides.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/crimson_qa_overrides.yaml
@@ -0,0 +1 @@
+.qa/config/crimson_qa_overrides.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/deploy/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/deploy/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/deploy/ceph.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/deploy/ceph.yaml
@@ -0,0 +1,11 @@
+overrides:
+  install:
+    ceph:
+      flavor: crimson
+tasks:
+- install:
+- ceph:
+    conf:
+      osd:
+        debug monc: 20
+    flavor: crimson
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/deploy/cephadm.yaml.disabled b/qa/suites/crimson-rados/thrash_seastore_radosbench/deploy/cephadm.yaml.disabled
@@ -0,0 +1,16 @@
+# no need to verify os + flavor + sha1
+verify_ceph_hash: false
+tasks:
+- cephadm:
+    conf:
+      mgr:
+        debug ms: 1
+        debug mgr: 20
+        debug osd: 10
+- cephadm.shell:
+    mon.a:
+      - ceph orch status
+      - ceph orch ps
+      - ceph orch ls
+      - ceph orch host ls
+      - ceph orch device ls
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/seastore/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/seastore/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/seastore/seastore-rbm.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/seastore/seastore-rbm.yaml
@@ -0,0 +1 @@
+../.qa/config/crimson_seastore_rbm.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/seastore/seastore-segmented.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/objectstore/seastore/seastore-segmented.yaml
@@ -0,0 +1 @@
+../.qa/config/crimson_seastore_segmented.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/thrashers/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/thrashers/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/thrashers/default.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/thrashers/default.yaml
@@ -0,0 +1,35 @@
+overrides:
+  ceph:
+    log-ignorelist:
+    - but it is still running
+    - objects unfound and apparently lost
+    conf:
+      osd:
+        osd debug reject backfill probability: .3
+        osd scrub min interval: 60
+        osd scrub max interval: 120
+        osd max backfills: 3
+        osd snap trim sleep: 2
+        osd delete sleep: 1
+      mon:
+        mon min osdmap epochs: 50
+        paxos service trim min: 10
+        # prune full osdmaps regularly
+        mon osdmap full prune min: 15
+        mon osdmap full prune interval: 2
+        mon osdmap full prune txsize: 2
+tasks:
+- thrashosds:
+    timeout: 2400
+    dump_ops_enable: false
+    sighup_delay: 0
+    min_in: 3
+    noscrub_toggle_delay: 0
+    chance_thrash_pg_upmap: 0
+    reweight_osd: 0
+    thrash_primary_affinity: false
+    ceph_objectstore_tool: false
+    chance_inject_pause_short: 0
+    chance_thrash_cluster_full: 0
+    chance_reset_purged_snaps_last: 0
+    chance_trim_stale_osdmaps: 0
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/thrashosds-health.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/thrashosds-health.yaml
@@ -0,0 +1 @@
+.qa/tasks/thrashosds-health.yaml
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/workloads/.qa b/qa/suites/crimson-rados/thrash_seastore_radosbench/workloads/.qa
@@ -0,0 +1 @@
+../.qa/
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/workloads/radosbench-high-concurrency.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/workloads/radosbench-high-concurrency.yaml
@@ -0,0 +1,49 @@
+overrides:
+  ceph:
+    conf:
+      client.0:
+        debug ms: 1
+        debug objecter: 20
+        debug rados: 20
+tasks:
+- full_sequential:
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      concurrency: 128
+      size: 8192
+      time: 90
diff --git a/qa/suites/crimson-rados/thrash_seastore_radosbench/workloads/radosbench.yaml b/qa/suites/crimson-rados/thrash_seastore_radosbench/workloads/radosbench.yaml
@@ -0,0 +1,24 @@
+overrides:
+  ceph:
+    conf:
+      client.0:
+        debug ms: 1
+        debug objecter: 20
+        debug rados: 20
+tasks:
+- full_sequential:
+  - radosbench:
+      clients: [client.0]
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      time: 90
+  - radosbench:
+      clients: [client.0]
+      time: 90
