Merge pull request ceph#60631 from kamoltat/wip-ksirivad-fix-3az-unset

[3AZ] src/mon : Allow user to specify crush_rule, size and min_size when unsetting a stretch pool

Author: SrinivasaBharath

doc/rados/operations/pools.rst

@@ -840,7 +840,7 @@ To move the pool back to non-stretch, run a command of the following form:
 
 .. prompt:: bash $
 
-   ceph osd pool stretch unset {pool-name}
+   ceph osd pool stretch unset {pool-name} {crush_rule} {size} {min_size}
 
 Here are the breakdowns of the arguments:
 
@@ -852,6 +852,28 @@ Here are the breakdowns of the arguments:
    :Type: String
    :Required: Yes.
 
+.. describe:: {crush_rule}
+      
+   The crush rule to use after exiting the stretch pool. The type of pool must match the type of crush_rule
+   (replicated or erasure).
+
+   :Type: String
+   :Required: Yes.
+
+.. describe:: {size}
+         
+   The number of replicas for objects after exiting stretch pool.
+   
+   :Type: Integer
+   :Required: Yes.
+
+.. describe:: {min_size}
+            
+   The minimum number of replicas required for I/O after exiting stretch pool.
+
+   :Type: Integer
+   :Required: Yes.
+
 Showing values of a stretch pool
 ================================
 To show values for a stretch pool, run a command of the following form:

qa/tasks/stretch_cluster.py

@@ -58,6 +58,7 @@ class TestStretchCluster(MgrTestCase):
     PEERING_CRUSH_BUCKET_TARGET = 3
     PEERING_CRUSH_BUCKET_BARRIER = 'datacenter'
     CRUSH_RULE = 'replicated_rule_custom'
+    DEFAULT_CRUSH_RULE = 'replicated_rule'
     SIZE = 6
     MIN_SIZE = 3
     BUCKET_MAX = SIZE // PEERING_CRUSH_BUCKET_TARGET
@@ -594,6 +595,17 @@ def test_mon_failures_in_stretch_pool(self):
             success_hold_time=self.SUCCESS_HOLD_TIME
         )
 
+        # Unset the pool back to replicated rule expects PGs to be 100% active+clean
+        self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'osd', 'pool', 'stretch', 'unset',
+            self.POOL, self.DEFAULT_CRUSH_RULE,
+            str(self.SIZE), str(self.MIN_SIZE))
+        self.wait_until_true_and_hold(
+            lambda: self._pg_all_active_clean(),
+            timeout=self.RECOVERY_PERIOD,
+            success_hold_time=self.SUCCESS_HOLD_TIME
+        )
+
     def test_set_stretch_pool_no_active_pgs(self):
         """
         Test setting a pool to stretch cluster and checks whether
@@ -686,10 +698,20 @@ def test_set_stretch_pool_no_active_pgs(self):
             timeout=self.RECOVERY_PERIOD,
             success_hold_time=self.SUCCESS_HOLD_TIME)
 
-        # Bring back osds iin DC2 expects PGs to be 100% active+clean
+        # Bring back osds in DC2 expects PGs to be 100% active+clean
         self._bring_back_all_osds_in_dc('dc2')
         self.wait_until_true_and_hold(
             lambda: self._pg_all_active_clean(),
             timeout=self.RECOVERY_PERIOD,
             success_hold_time=self.SUCCESS_HOLD_TIME
         )
+        # Unset the pool back to replicated rule expects PGs to be 100% active+clean
+        self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'osd', 'pool', 'stretch', 'unset',
+            self.POOL, self.DEFAULT_CRUSH_RULE,
+            str(self.SIZE), str(self.MIN_SIZE))
+        self.wait_until_true_and_hold(
+            lambda: self._pg_all_active_clean(),
+            timeout=self.RECOVERY_PERIOD,
+            success_hold_time=self.SUCCESS_HOLD_TIME
+        )

qa/tasks/test_netsplit_3az_stretch_pool.py

@@ -21,6 +21,7 @@ class TestNetSplit(CephTestCase):
     PEERING_CRUSH_BUCKET_BARRIER = 'datacenter'
     POOL = 'pool_stretch'
     CRUSH_RULE = 'replicated_rule_custom'
+    DEFAULT_CRUSH_RULE = 'replicated_rule'
     SIZE = 6
     MIN_SIZE = 3
     BUCKET_MAX = SIZE // PEERING_CRUSH_BUCKET_TARGET
@@ -278,4 +279,14 @@ def test_mon_netsplit(self):
             timeout=self.RECOVERY_PERIOD,
             success_hold_time=self.SUCCESS_HOLD_TIME
         )
+        # Unset the pool back to replicated rule expects PGs to be 100% active+clean
+        self.mgr_cluster.mon_manager.raw_cluster_cmd(
+            'osd', 'pool', 'stretch', 'unset',
+            self.POOL, self.DEFAULT_CRUSH_RULE,
+            str(self.SIZE), str(self.MIN_SIZE))
+        self.wait_until_true_and_hold(
+            lambda: self._pg_all_active_clean(),
+            timeout=self.RECOVERY_PERIOD,
+            success_hold_time=self.SUCCESS_HOLD_TIME
+        )
         log.info("test_mon_netsplit passed!")

qa/workunits/mon/mon-stretch-pool.sh

@@ -97,10 +97,12 @@ expect_false ceph osd pool stretch set non_exist_pool 2 3 datacenter $TEST_CRUSH
 expect_false ceph osd pool stretch set $TEST_POOL_STRETCH 2 3 non_exist_barrier $TEST_CRUSH_RULE 6 3
 # Non existence crush_rule should return appropriate error
 expect_false ceph osd pool stretch set $TEST_POOL_STRETCH 2 3 datacenter $TEST_CRUSH_RULE 6 3
+# Unsetting a pool with missing arguments
+expect_false ceph osd pool stretch unset $TEST_POOL_STRETCH
 # Unsetting a non existence pool should return error
-expect_false ceph osd pool stretch unset non_exist_pool
+expect_false ceph osd pool stretch unset non_exist_pool replicated_rule 6 3
 # Unsetting a non-stretch pool should return error
-expect_false ceph osd pool stretch unset $TEST_POOL_STRETCH
+expect_false ceph osd pool stretch unset $TEST_POOL_STRETCH replicated_rule 6 3
 
 # Create a custom crush rule
 ceph osd getcrushmap > crushmap
@@ -139,7 +141,7 @@ expect_true ceph osd pool stretch set $TEST_POOL_STRETCH 2 3 datacenter $TEST_CR
 expect_true ceph osd pool stretch show $TEST_POOL_STRETCH
 
 # Unset the stretch pool and expects it to work
-expect_true ceph osd pool stretch unset $TEST_POOL_STRETCH
+expect_true ceph osd pool stretch unset $TEST_POOL_STRETCH replicated_rule 6 3
 # try to show the stretch pool values again, should return error since
 # the pool is not a stretch pool anymore.
 expect_false ceph osd pool stretch show $TEST_POOL_STRETCH

src/mon/MonCommands.h

@@ -1237,7 +1237,10 @@ COMMAND("osd pool stretch set "
         "make the pool stretched across the specified number of CRUSH buckets",
         "osd", "rw")
 COMMAND("osd pool stretch unset "
-		"name=pool,type=CephPoolname",
+		"name=pool,type=CephPoolname "
+		"name=crush_rule,type=CephString "
+		"name=size,type=CephInt,range=0 "
+		"name=min_size,type=CephInt,range=0 ",
 		"unset the stretch mode for the pool",
 		"osd", "rw")
 COMMAND("osd utilization",

src/mon/OSDMonitor.cc

@@ -9237,11 +9237,44 @@ int OSDMonitor::prepare_command_pool_stretch_unset(const cmdmap_t& cmdmap,
     ss << "pool " << pool_name << " is not a stretch pool";
     return -ENOENT;
   }
+  CrushWrapper& crush = _get_stable_crush();
+  string crush_rule_str;
+  cmd_getval(cmdmap, "crush_rule", crush_rule_str);
+  if (crush_rule_str.empty()) {
+    ss << "crush_rule must be provided";
+    return -EINVAL;
+  }
+
+  int crush_rule = crush.get_rule_id(crush_rule_str);
+  if (crush_rule < 0) {
+    ss << "crush rule " << crush_rule_str << " does not exist";
+    return -ENOENT;
+  }
+
+  if (!crush.rule_valid_for_pool_type(crush_rule, p.get_type())) {
+    ss << "crush rule " << crush_rule << " type does not match pool";
+    return -EINVAL;
+  }
+
+  int64_t pool_size = cmd_getval_or<int64_t>(cmdmap, "size", 0);
+  if (pool_size < 0) {
+    ss << "pool size must be non-negative";
+    return -EINVAL;
+  }
+
+  int64_t pool_min_size = cmd_getval_or<int64_t>(cmdmap, "min_size", 0);
+  if (pool_min_size < 0) {
+    ss << "pool min_size must be non-negative";
+    return -EINVAL;
+  }
 
   // unset stretch values
   p.peering_crush_bucket_count = 0;
   p.peering_crush_bucket_target = 0;
   p.peering_crush_bucket_barrier = 0;
+  p.crush_rule = static_cast<__u8>(crush_rule);
+  p.size = static_cast<__u8>(pool_size);
+  p.min_size = static_cast<__u8>(pool_min_size);
   p.last_change = pending_inc.epoch;
   pending_inc.new_pools[pool] = p;
   ss << "pool " << pool_name