CA-384228: Add MTU diagnostics during pool join

Add diagnostic tests during pool join to detect and warn about MTU
mismatches, particularly when jumbo frames are configured but the
network path doesn't support them.

The diagnostics:
1. Query master's management network MTU via RPC
2. Test standard MTU (1472 bytes data) with ICMP ping
3. Test jumbo frames (8972 bytes data) if MTU > 1500
4. Log prominent warning when CA-384228 scenario detected:
   - Standard MTU works
   - Jumbo frames fail
   - This indicates path MTU < configured MTU

Key design decisions:
- Does NOT block pool join (ICMP may be blocked by firewalls)
- Queries master's DB (slave's DB not yet synced during join)
- Called after RPC session established (need remote DB access)
- Relies on TCP PMTUD to handle issues automatically
- Diagnostics are informational only for visibility

Warning format highlights the issue clearly and references the
TCP PMTUD fix that handles it automatically, with guidance for
persistent problems.

Signed-off-by: Gang Ji <gang.ji@cloud.com>

Author: gangj

ocaml/xapi/xapi_pool.ml

@@ -62,6 +62,113 @@ let get_master ~rpc ~session_id =
   let pool = get_pool ~rpc ~session_id in
   Client.Pool.get_master ~rpc ~session_id ~self:pool
 
+(* MTU diagnostics during pool join - CA-384228
+ *
+ * This provides visibility into MTU issues but does NOT block pool join because:
+ * 1. ICMP may be blocked by firewalls, causing false negatives
+ * 2. TCP PMTUD (net.ipv4.tcp_mtu_probing=1) is now enabled by default and handles
+ *    MTU mismatches automatically at the TCP layer
+ * 3. TCP PMTUD works even when ICMP is blocked (detects via packet loss + retries)
+ *)
+let check_mtu_connectivity ~__context ~rpc ~session_id ~master_address
+    ~master_host =
+  (* Query the master's management PIF to get the actual configured MTU *)
+  let master_mgmt_pif =
+    Client.Host.get_management_interface ~rpc ~session_id ~host:master_host
+  in
+  let master_network =
+    Client.PIF.get_network ~rpc ~session_id ~self:master_mgmt_pif
+  in
+  let configured_mtu =
+    Client.Network.get_MTU ~rpc ~session_id ~self:master_network
+  in
+  let is_jumbo = configured_mtu > 1500L in
+
+  debug
+    "MTU diagnostics: configured MTU=%Ld on master's management network to \
+     master %s. TCP PMTUD enabled via sysctl - will auto-adjust if path MTU is \
+     smaller"
+    configured_mtu master_address ;
+
+  (* ICMP payload sizes for MTU testing:
+     - 1472 = 1500 (standard MTU) - 20 (IP header) - 8 (ICMP header)
+     - 8972 = 9000 (jumbo MTU) - 20 (IP header) - 8 (ICMP header) *)
+  let standard_mtu_icmp_payload = 1472 in
+  let jumbo_mtu_icmp_payload = 8972 in
+
+  (* Test MTU connectivity using ping - ICMP-based, informational only *)
+  let test_ping size desc =
+    try
+      let timeout = 3.0 *. 1e9 |> Int64.of_float |> Mtime.Span.of_uint64_ns in
+      let _stdout, _stderr =
+        Forkhelpers.execute_command_get_output ~timeout "/usr/bin/ping"
+          [
+            "-c"
+          ; "3"
+          ; "-M"
+          ; "do"
+          ; "-s"
+          ; string_of_int size
+          ; "-W"
+          ; "1"
+          ; master_address
+          ]
+      in
+      debug "MTU diagnostics: %s test PASSED (size=%d)" desc size ;
+      true
+    with e ->
+      debug "MTU diagnostics: %s test FAILED (size=%d): %s" desc size
+        (ExnHelper.string_of_exn e) ;
+      false
+  in
+
+  let standard_ok = test_ping standard_mtu_icmp_payload "standard MTU" in
+
+  (* Check MTU connectivity and report results *)
+  if is_jumbo then
+    let jumbo_ok = test_ping jumbo_mtu_icmp_payload "jumbo frame" in
+    match (standard_ok, jumbo_ok) with
+    | true, false ->
+        (* CA-384228 scenario: standard works but jumbo fails *)
+        warn
+          "MTU diagnostics: MTU CONFIGURATION ISSUE DETECTED (CA-384228): \
+           Jumbo frames (MTU %Ld) configured but network path does not support \
+           them! Standard MTU works, but jumbo frames fail. This can cause TCP \
+           connection hangs during pool operations with large requests. TCP \
+           PMTUD (net.ipv4.tcp_mtu_probing=1) is enabled and should handle \
+           this automatically, but if you experience hangs, consider reducing \
+           MTU to 1500 or fixing network infrastructure. Reference: \
+           https://blog.cloudflare.com/path-mtu-discovery-in-practice/"
+          configured_mtu
+    | false, false ->
+        (* Both tests failed - ICMP may be blocked *)
+        warn
+          "MTU diagnostics: Both standard and jumbo MTU tests failed (ICMP may \
+           be blocked). If ICMP is blocked, ignore this - TCP PMTUD will \
+           handle it. If ICMP is NOT blocked, check network connectivity to \
+           master %s"
+          master_address
+    | false, true ->
+        (* Unusual: standard failed but jumbo passed *)
+        warn
+          "MTU diagnostics: Unusual result - standard MTU failed but jumbo \
+           frames passed (likely ICMP issue). TCP PMTUD will handle this - \
+           monitor for issues"
+    | true, true ->
+        (* Both tests passed - ideal case *)
+        debug
+          "MTU diagnostics: Both standard and jumbo frame tests passed - \
+           network path fully supports configured MTU"
+  else if not standard_ok then
+    warn
+      "MTU diagnostics: Standard MTU test failed (ICMP may be blocked or \
+       connectivity issue to %s). TCP PMTUD will handle this - monitor for \
+       issues"
+      master_address
+  else
+    debug
+      "MTU diagnostics: Standard MTU test passed, no jumbo frames configured"
+
 (* Pre-join asserts *)
 let pre_join_checks ~__context ~rpc ~session_id ~force =
   (* I cannot join a Pool unless my management interface exists in the db, otherwise
@@ -1631,6 +1738,7 @@ let join_common ~__context ~master_address ~master_username ~master_password
          side. If we're trying to join a host that does not support pooling
          then an error will be thrown at this stage *)
       pre_join_checks ~__context ~rpc:unverified_rpc ~session_id ~force ;
+
       (* get hold of cluster secret - this is critical; if this fails whole pool join fails *)
       new_pool_secret :=
         Client.Pool.initial_auth ~rpc:unverified_rpc ~session_id ;
@@ -1665,6 +1773,10 @@ let join_common ~__context ~master_address ~master_username ~master_password
   in
 
   let remote_coordinator = get_master ~rpc ~session_id in
+
+  check_mtu_connectivity ~__context ~rpc ~session_id ~master_address
+    ~master_host:remote_coordinator ;
+
   (* If management is on a VLAN, then get the Pool master
      management network bridge before we logout the session *)
   let pool_master_bridge, mgmt_pif =