Merge pull request ceph#64122 from kamoltat/wip-ksirivad-fix-msg-v2

msg/async/ProtocolV2: Speed up connection logic when peer restarts & Added Netsplit Grace Period

Author: ljflores

doc/rados/operations/health-checks.rst

@@ -167,6 +167,12 @@ which are frequently updated. This warning only appears when
 the cluster is provisioned with at least three Ceph Monitors and are using the
 ``connectivity`` election strategy.
 
+To reduce false alarms from transient network issues, detected netsplits are
+not immediately reported as health warnings. Instead, they must persist for at
+least ``mon_netsplit_grace_period`` seconds (default: 9 seconds) before being
+reported. If the network partition resolves within this grace period, no health
+warning is emitted.
+
 Network partitions are reported in two ways:
 
 - As location-level netsplits (e.g., "Netsplit detected between dc1 and dc2") when
@@ -177,6 +183,18 @@ Network partitions are reported in two ways:
 The system prioritizes reporting at the highest topology level (``datacenter``, ``rack``, etc.)
 when possible, to better help operators identify infrastructure-level network issues.
 
+To adjust the grace period threshold, run the following command:
+
+.. prompt:: bash $
+
+   ceph config set mon mon_netsplit_grace_period <seconds>
+
+To disable the grace period entirely (immediate reporting), set the value to 0:
+
+.. prompt:: bash $
+
+   ceph config set mon mon_netsplit_grace_period 0
+
 AUTH_INSECURE_GLOBAL_ID_RECLAIM
 _______________________________
 

qa/config/rados.yaml

@@ -11,3 +11,4 @@ overrides:
         osd mclock profile: high_recovery_ops
       mon:
         mon scrub interval: 300
+        debug mon: 30

qa/suites/netsplit/ceph.yaml

@@ -31,6 +31,7 @@ overrides:
       - \(PG_AVAILABILITY\)
       - \(SLOW_OPS\)
       - \[WRN\]
+      - \(MON_NETSPLIT\)
 tasks:
 - install:
 - ceph:

qa/suites/rados/monthrash/ceph.yaml

@@ -13,6 +13,7 @@ overrides:
         mon osdmap full prune txsize: 2
         mon scrub inject crc mismatch: 0.01
         mon scrub inject missing keys: 0.05
+        debug ms: 20
 # thrashing monitors may make mgr have trouble w/ its keepalive
     log-ignorelist:
       - ScrubResult

qa/suites/rados/monthrash/msgr-failures/few.yaml

@@ -6,3 +6,4 @@ overrides:
         mon client directed command retry: 5
     log-ignorelist:
       - \(OSD_SLOW_PING_TIME
+      - \(MON_NETSPLIT)

qa/suites/rados/monthrash/msgr-failures/mon-delay.yaml

@@ -12,3 +12,4 @@ overrides:
         debug monc: 10
     log-ignorelist:
       - \(OSD_SLOW_PING_TIME
+      - \(MON_NETSPLIT)

qa/suites/rados/monthrash/msgr-failures/none.yaml

@@ -0,0 +1,6 @@
+overrides:
+  ceph:
+    conf:
+      global:
+        ms inject socket failures: 0
+

qa/suites/rados/multimon/ceph.yaml

@@ -0,0 +1,6 @@
+overrides:
+  ceph:
+    conf:
+      mon:
+        debug ms: 20
+

qa/suites/rados/multimon/msgr-failures/few.yaml

@@ -6,3 +6,4 @@ overrides:
         mon client directed command retry: 5
     log-ignorelist:
       - \(OSD_SLOW_PING_TIME
+      - \(MON_NETSPLIT)

qa/suites/rados/multimon/msgr-failures/many.yaml

@@ -7,3 +7,4 @@ overrides:
         mon mgr beacon grace: 90
     log-ignorelist:
       - \(OSD_SLOW_PING_TIME
+      - \(MON_NETSPLIT)