Merge pull request ceph#57906 from kamoltat/wip-ksirivad-stretch-mode-netsplit-test

mon/ElectionLogic: tie-breaker mon ignore proposal from marked down mon

Reviewed-by: Greg Farnum <[email protected]>

Author: yuriw

qa/suites/netsplit/ceph.yaml

@@ -10,14 +10,27 @@ overrides:
         mon osdmap full prune min: 15
         mon osdmap full prune interval: 2
         mon osdmap full prune txsize: 2
+        debug mon: 30
 # thrashing monitors may make mgr have trouble w/ its keepalive
     log-ignorelist:
+      - overall HEALTH_
+      - \(OSDMAP_FLAGS\)
+      - \(OSD_
+      - \(PG_
+      - \(POOL_
+      - \(CACHE_POOL_
+      - \(OBJECT_
+      - \(SLOW_OPS\) # slow mons -> slow peering -> PG_AVAILABILITY
+      - \(REQUEST_SLOW\)
+      - \(TOO_FEW_PGS\)
+      - slow request
+      - \(POOL_APP_NOT_ENABLED\)
       - overall HEALTH_
       - \(MGR_DOWN\)
       - \(MON_DOWN\)
-# slow mons -> slow peering -> PG_AVAILABILITY
       - \(PG_AVAILABILITY\)
       - \(SLOW_OPS\)
+      - \[WRN\]
 tasks:
 - install:
 - ceph:

qa/suites/netsplit/cluster.yaml

@@ -1,7 +1,8 @@
 roles:
-- [mon.a, mgr.x, osd.0, osd.1, osd.2, osd.3]
-- [mon.b, mgr.y, osd.4, osd.5, osd.6, osd.7, client.0]
-- [mon.c]
+- [mon.a, mon.b, mgr.x, mds.a, osd.0, osd.1, osd.2, osd.3]
+- [mon.c, mon.d, mgr.y, mds.b, osd.4, osd.5, osd.6, osd.7]
+- [mon.e, mgr.z, mds.c]
+- [client.0]
 openstack:
 - volumes: # attached to each instance
     count: 4

qa/suites/netsplit/tests/2-dc-1-arbiter-stretch-cluster-netsplit.yaml

@@ -0,0 +1,13 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        mon election default strategy: 3
+tasks:
+- workunit:
+    clients:
+      client.0:
+        - mon/setup_stretch_cluster.sh
+- cephfs_test_runner:
+    modules:
+      - tasks.test_netsplit

qa/suites/netsplit/tests/mon_pool_ops.yaml

@@ -341,3 +341,37 @@ def wait_until_true(cls, condition, timeout, check_fn=None, period=5):
                     log.debug("wait_until_true: waiting (timeout={0} retry_count={1})...".format(timeout, retry_count))
                 time.sleep(period)
                 elapsed += period
+
+    @classmethod
+    def wait_until_true_and_hold(cls, condition, timeout, success_hold_time, check_fn=None, period=5):
+        """
+        Wait until the condition is met and check if the condition holds for the remaining time.
+        """
+        elapsed = 0
+        retry_count = 0
+        assert success_hold_time < timeout, "success_hold_time should not be greater than timeout"
+        while True:
+            if condition():
+                success_time_elapsed = 0
+                while success_time_elapsed < success_hold_time:
+                    if condition():
+                        success_time_elapsed += 1
+                        time.sleep(1)
+                        elapsed += 1
+                    else:
+                        break
+                if success_time_elapsed == success_hold_time:
+                    log.debug("wait_until_true_and_hold: success for {0}s".format(success_hold_time))
+                    return
+            else:
+                if elapsed >= timeout:
+                    if check_fn and check_fn() and retry_count < 5:
+                        elapsed = 0
+                        retry_count += 1
+                        log.debug("wait_until_true_and_hold: making progress, waiting (timeout={0} retry_count={1})...".format(timeout, retry_count))
+                    else:
+                        raise TestTimeoutError("Timed out after {0}s and {1} retries".format(elapsed, retry_count))
+                else:
+                    log.debug("wait_until_true_and_hold waiting (timeout={0} retry_count={1})...".format(timeout, retry_count))
+                time.sleep(period)
+                elapsed += period

qa/tasks/ceph_test_case.py

@@ -12,7 +12,11 @@
 
 log = logging.getLogger(__name__)
 
+
 def get_ip_and_ports(ctx, daemon):
+    """
+    Get the IP and port list for the <daemon>.
+    """
     assert daemon.startswith('mon.')
     addr = ctx.ceph['ceph'].mons['{a}'.format(a=daemon)]
     ips = re.findall("[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+[:[0-9]*]*", addr)
@@ -27,11 +31,16 @@ def get_ip_and_ports(ctx, daemon):
             port_list.append(port_str)
     return (plain_ip, port_list)
 
+
 def disconnect(ctx, config):
-    assert len(config) == 2 # we can only disconnect pairs right now
+    """
+    Disconnect the mons in the <config> list.
+    """
+    assert len(config) == 2  # we can only disconnect pairs right now
     # and we can only disconnect mons right now
     assert config[0].startswith('mon.')
     assert config[1].startswith('mon.')
+    log.info("Disconnecting {a} and {b}".format(a=config[0], b=config[1]))
     (ip1, _) = get_ip_and_ports(ctx, config[0])
     (ip2, _) = get_ip_and_ports(ctx, config[1])
 
@@ -40,21 +49,26 @@ def disconnect(ctx, config):
     assert host1 is not None
     assert host2 is not None
 
-    host1.run(
-        args = ["sudo", "iptables", "-A", "INPUT", "-p", "tcp", "-s",
-                ip2, "-j", "DROP"]
-    )
-    host2.run(
-        args = ["sudo", "iptables", "-A", "INPUT", "-p", "tcp", "-s",
-                ip1, "-j", "DROP"]
-    )
+    host1.run(args=["sudo", "iptables", "-A", "INPUT",
+                    "-s", ip2, "-j", "DROP"])
+    host1.run(args=["sudo", "iptables", "-A", "OUTPUT",
+                    "-d", ip2, "-j", "DROP"])
+
+    host2.run(args=["sudo", "iptables", "-A", "INPUT",
+                    "-s", ip1, "-j", "DROP"])
+    host2.run(args=["sudo", "iptables", "-A", "OUTPUT",
+                    "-d", ip1, "-j", "DROP"])
+
 
 def reconnect(ctx, config):
-    assert len(config) == 2 # we can only disconnect pairs right now
+    """
+    Reconnect the mons in the <config> list.
+    """
+    assert len(config) == 2  # we can only disconnect pairs right now
     # and we can only disconnect mons right now
     assert config[0].startswith('mon.')
     assert config[1].startswith('mon.')
-
+    log.info("Reconnecting {a} and {b}".format(a=config[0], b=config[1]))
     (ip1, _) = get_ip_and_ports(ctx, config[0])
     (ip2, _) = get_ip_and_ports(ctx, config[1])
 
@@ -63,11 +77,12 @@ def reconnect(ctx, config):
     assert host1 is not None
     assert host2 is not None
 
-    host1.run(
-        args = ["sudo", "iptables", "-D", "INPUT", "-p", "tcp", "-s",
-                ip2, "-j", "DROP"]
-    )
-    host2.run(
-        args = ["sudo", "iptables", "-D", "INPUT", "-p", "tcp", "-s",
-                ip1, "-j", "DROP"]
-    )
+    host1.run(args=["sudo", "iptables", "-D", "INPUT",
+                    "-s", ip2, "-j", "DROP"])
+    host1.run(args=["sudo", "iptables", "-D", "OUTPUT",
+                    "-d", ip2, "-j", "DROP"])
+
+    host2.run(args=["sudo", "iptables", "-D", "INPUT",
+                    "-s", ip1, "-j", "DROP"])
+    host2.run(args=["sudo", "iptables", "-D", "OUTPUT",
+                    "-d", ip1, "-j", "DROP"])