Add cluster ID mismatch to connectivity messages (#2045)

v0d1ch · web-flow · commit 6b38190534a6 · 2025-06-04T14:46:23.000+02:00
fix #2026 ---  * [x] CHANGELOG updated or not needed * [x] Documentation updated or not needed * [x] Haddocks updated or not needed * [x] No new TODOs introduced or explained herafter
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -43,6 +43,9 @@ changes.
   - This introduces `ToJSON` and `FromJSON` instances on all observation types.
   - `CollectComObservation` and `ContestObservation` are made compatible with their `OnChainTx` counterparts.
 
+- Enhanced the error message for `etcd` cluster ID mismatches by including detailed information about
+  the expected peers versus peers loaded from the `hydra-node` arguments.
+
 ## [0.21.0] - 2025-04-28
 
 - New metric for counting the number of active peers: `hydra_head_peers_connected`
diff --git a/docs/docs/known-issues.md b/docs/docs/known-issues.md
@@ -28,7 +28,9 @@ The network topology needs to be statically configured and match across all `hyd
 Known errors are:
 
  - `cluster ID mismatch` - the cluster was initiated with a different list of `--peer`s
-   - check configuration with other participants
+   - check configuration with other participants. There should be a corresponding log entry `NetworkClusterIDMismatch` with the information on:
+      - `clusterPeers` - loaded peers info received from `etcd` cluster.
+      - `configuredPeers` - peers info coming from `hydra-node` arguments.
 
  - `member ... has already been bootstrapped` - missing information in `<persistence-dir>/etcd`
    - need to bootstrap new cluster or manual workarounds, see also https://etcd.io/docs/v3.5/op-guide/failures/
diff --git a/hydra-cluster/src/Hydra/Cluster/Scenarios.hs b/hydra-cluster/src/Hydra/Cluster/Scenarios.hs
@@ -1172,6 +1172,31 @@ initWithWrongKeys workDir tracer node@RunningNode{nodeSocket} hydraScriptsTxId =
 
       participants `shouldMatchList` expectedParticipants
 
+startWithWrongPeers :: FilePath -> Tracer IO EndToEndLog -> RunningNode -> [TxId] -> IO ()
+startWithWrongPeers workDir tracer node@RunningNode{nodeSocket} hydraScriptsTxId = do
+  (aliceCardanoVk, _) <- keysFor Alice
+
+  let contestationPeriod = 2
+  aliceChainConfig <- chainConfigFor Alice workDir nodeSocket hydraScriptsTxId [Carol] contestationPeriod
+  bobChainConfig <- chainConfigFor Bob workDir nodeSocket hydraScriptsTxId [Alice] contestationPeriod
+
+  let hydraTracer = contramap FromHydraNode tracer
+  withHydraNode hydraTracer aliceChainConfig workDir 3 aliceSk [bobVk] [3, 4] $ \n1 -> do
+    -- NOTE: here we deliberately use the wrong peer list for Bob
+    withHydraNode hydraTracer bobChainConfig workDir 4 bobSk [aliceVk] [4] $ \_ -> do
+      seedFromFaucet_ node aliceCardanoVk 100_000_000 (contramap FromFaucet tracer)
+
+      (clusterPeers, configuredPeers) <- waitMatch 20 n1 $ \v -> do
+        guard $ v ^? key "tag" == Just (Aeson.String "NetworkClusterIDMismatch")
+        clusterPeers <- v ^? key "clusterPeers" . _String
+        configuredPeers <- v ^? key "misconfiguredPeers" . _String
+        pure (clusterPeers, configuredPeers)
+
+      when (clusterPeers == configuredPeers) $
+        failure "Expected clusterPeers and configuredPeers to be different"
+      clusterPeers `shouldBe` "0.0.0.0:5003=http://0.0.0.0:5003,0.0.0.0:5004=http://0.0.0.0:5004"
+      configuredPeers `shouldBe` "0.0.0.0:5004=http://0.0.0.0:5004"
+
 -- | Open a a two participant head and incrementally commit to it.
 canCommit :: Tracer IO EndToEndLog -> FilePath -> RunningNode -> [TxId] -> IO ()
 canCommit tracer workDir node hydraScriptsTxId =
diff --git a/hydra-cluster/test/Test/EndToEndSpec.hs b/hydra-cluster/test/Test/EndToEndSpec.hs
@@ -75,6 +75,7 @@ import Hydra.Cluster.Scenarios (
   singlePartyHeadFullLifeCycle,
   singlePartyUsesScriptOnL2,
   singlePartyUsesWithdrawZeroTrick,
+  startWithWrongPeers,
   threeNodesNoErrorsOnOpen,
   threeNodesWithMirrorParty,
  )
@@ -566,6 +567,13 @@ spec = around (showLogsOnFailure "EndToEndSpec") $ do
               publishHydraScriptsAs node Faucet
                 >>= initWithWrongKeys tmpDir tracer node
 
+      it "cluster id mismatch provides useful info in the logs" $ \tracer ->
+        failAfter 60 $
+          withClusterTempDir $ \tmpDir -> do
+            withCardanoNodeDevnet (contramap FromCardanoNode tracer) tmpDir $ \node -> do
+              publishHydraScriptsAs node Faucet
+                >>= startWithWrongPeers tmpDir tracer node
+
       it "bob cannot abort alice's head" $ \tracer -> do
         failAfter 60 $
           withClusterTempDir $ \tmpDir -> do
diff --git a/hydra-node/golden/ServerOutput/NetworkClusterIDMismatch.json b/hydra-node/golden/ServerOutput/NetworkClusterIDMismatch.json
@@ -0,0 +1,10 @@
+{
+    "samples": [
+        {
+            "clusterPeers": "",
+            "misconfiguredPeers": "",
+            "tag": "NetworkClusterIDMismatch"
+        }
+    ],
+    "seed": 1341180695
+}
diff --git a/hydra-node/golden/StateChanged/NetworkClusterIDMismatch.json b/hydra-node/golden/StateChanged/NetworkClusterIDMismatch.json
@@ -0,0 +1,10 @@
+{
+    "samples": [
+        {
+            "clusterPeers": "",
+            "misconfiguredPeers": "$",
+            "tag": "NetworkClusterIDMismatch"
+        }
+    ],
+    "seed": 1702753294
+}
diff --git a/hydra-node/json-schemas/api.yaml b/hydra-node/json-schemas/api.yaml
@@ -80,6 +80,7 @@ channels:
           - $ref: "api.yaml#/components/messages/NetworkConnected"
           - $ref: "api.yaml#/components/messages/NetworkDisconnected"
           - $ref: "api.yaml#/components/messages/NetworkVersionMismatch"
+          - $ref: "api.yaml#/components/messages/NetworkClusterIDMismatch"
           - $ref: "api.yaml#/components/messages/HeadIsInitializing"
           - $ref: "api.yaml#/components/messages/Committed"
           - $ref: "api.yaml#/components/messages/HeadIsOpen"
@@ -487,6 +488,13 @@ components:
       payload:
         $ref: "api.yaml#/components/schemas/NetworkVersionMismatch"
 
+    NetworkClusterIDMismatch:
+      title: NetworkClusterIDMismatch
+      description: |
+        The configured peers do not match with other nodes on the network.
+      payload:
+        $ref: "api.yaml#/components/schemas/NetworkClusterIDMismatch"
+
     HeadIsInitializing:
       title: HeadIsInitializing
       description: |
@@ -732,6 +740,7 @@ components:
         - $ref: "api.yaml#/components/schemas/NetworkConnected"
         - $ref: "api.yaml#/components/schemas/NetworkDisconnected"
         - $ref: "api.yaml#/components/schemas/NetworkVersionMismatch"
+        - $ref: "api.yaml#/components/schemas/NetworkClusterIDMismatch"
         - $ref: "api.yaml#/components/schemas/PeerConnected"
         - $ref: "api.yaml#/components/schemas/PeerDisconnected"
         - $ref: "api.yaml#/components/schemas/HeadIsInitializing"
@@ -985,6 +994,25 @@ components:
         timestamp:
           $ref: "api.yaml#/components/schemas/UTCTime"
 
+    NetworkClusterIDMismatch:
+      type: object
+      required:
+        - tag
+        - seq
+        - timestamp
+      properties:
+        tag:
+          type: string
+          enum: ["NetworkClusterIDMismatch"]
+        clusterPeers:
+          type: string
+        misconfiguredPeers:
+          type: string
+        seq:
+          $ref: "api.yaml#/components/schemas/SequenceNumber"
+        timestamp:
+          $ref: "api.yaml#/components/schemas/UTCTime"
+
     HeadIsInitializing:
       type: object
       required:
diff --git a/hydra-node/src/Hydra/API/Server.hs b/hydra-node/src/Hydra/API/Server.hs
@@ -249,6 +249,7 @@ mkTimedServerOutputFromStateEvent event =
     StateChanged.NetworkConnected -> Just NetworkConnected
     StateChanged.NetworkDisconnected -> Just NetworkDisconnected
     StateChanged.NetworkVersionMismatch{..} -> Just NetworkVersionMismatch{..}
+    StateChanged.NetworkClusterIDMismatch{..} -> Just NetworkClusterIDMismatch{..}
     StateChanged.PeerConnected{..} -> Just PeerConnected{..}
     StateChanged.PeerDisconnected{..} -> Just PeerDisconnected{..}
     StateChanged.TransactionReceived{} -> Nothing
diff --git a/hydra-node/src/Hydra/API/ServerOutput.hs b/hydra-node/src/Hydra/API/ServerOutput.hs
@@ -142,6 +142,10 @@ data ServerOutput tx
       { ourVersion :: ProtocolVersion
       , theirVersion :: Maybe ProtocolVersion
       }
+  | NetworkClusterIDMismatch
+      { clusterPeers :: Text
+      , misconfiguredPeers :: Text
+      }
   | PeerConnected {peer :: Host}
   | PeerDisconnected {peer :: Host}
   | HeadIsInitializing {headId :: HeadId, parties :: [Party]}
@@ -273,6 +277,7 @@ prepareServerOutput config response =
     NetworkConnected -> encodedResponse
     NetworkDisconnected -> encodedResponse
     NetworkVersionMismatch{} -> encodedResponse
+    NetworkClusterIDMismatch{} -> encodedResponse
     PeerConnected{} -> encodedResponse
     PeerDisconnected{} -> encodedResponse
     SnapshotSideLoaded{} -> encodedResponse
diff --git a/hydra-node/src/Hydra/HeadLogic.hs b/hydra-node/src/Hydra/HeadLogic.hs
@@ -103,14 +103,16 @@ import Hydra.Tx.OnChainId (OnChainId)
 import Hydra.Tx.Party (Party (vkey))
 import Hydra.Tx.Snapshot (ConfirmedSnapshot (..), Snapshot (..), SnapshotNumber, SnapshotVersion, getSnapshot)
 
-onConnectionEvent :: Network.Connectivity -> Outcome tx
-onConnectionEvent = \case
+onConnectionEvent :: Text -> Network.Connectivity -> Outcome tx
+onConnectionEvent misconfiguredPeers = \case
   Network.NetworkConnected ->
     newState NetworkConnected
   Network.NetworkDisconnected ->
     newState NetworkDisconnected
   Network.VersionMismatch{ourVersion, theirVersion} ->
     newState NetworkVersionMismatch{ourVersion, theirVersion}
+  Network.ClusterIDMismatch{clusterPeers} ->
+    newState NetworkClusterIDMismatch{clusterPeers, misconfiguredPeers}
   Network.PeerConnected{peer} ->
     newState PeerConnected{peer}
   Network.PeerDisconnected{peer} ->
@@ -1313,7 +1315,7 @@ update ::
   Outcome tx
 update env ledger st ev = case (st, ev) of
   (_, NetworkInput _ (ConnectivityEvent conn)) ->
-    onConnectionEvent conn
+    onConnectionEvent env.configuredPeers conn
   (Idle _, ClientInput Init) ->
     onIdleClientInit env
   (Idle _, ChainInput Observation{observedTx = OnInitTx{headId, headSeed, headParameters, participants}, newChainState}) ->
@@ -1426,6 +1428,7 @@ aggregate st = \case
   NetworkConnected -> st
   NetworkDisconnected -> st
   NetworkVersionMismatch{} -> st
+  NetworkClusterIDMismatch{} -> st
   PeerConnected{} -> st
   PeerDisconnected{} -> st
   HeadInitialized{parameters = parameters@HeadParameters{parties}, headId, headSeed, chainState} ->
@@ -1793,6 +1796,7 @@ aggregateChainStateHistory history = \case
   NetworkConnected -> history
   NetworkDisconnected -> history
   NetworkVersionMismatch{} -> history
+  NetworkClusterIDMismatch{} -> history
   PeerConnected{} -> history
   PeerDisconnected{} -> history
   HeadInitialized{chainState} -> pushNewState chainState history
diff --git a/hydra-node/src/Hydra/HeadLogic/Outcome.hs b/hydra-node/src/Hydra/HeadLogic/Outcome.hs
@@ -60,6 +60,10 @@ data StateChanged tx
       { ourVersion :: ProtocolVersion
       , theirVersion :: Maybe ProtocolVersion
       }
+  | NetworkClusterIDMismatch
+      { clusterPeers :: Text
+      , misconfiguredPeers :: Text
+      }
   | HeadInitialized
       { parameters :: HeadParameters
       , chainState :: ChainStateType tx
diff --git a/hydra-node/src/Hydra/Network.hs b/hydra-node/src/Hydra/Network.hs
@@ -185,6 +185,9 @@ data Connectivity
       { ourVersion :: ProtocolVersion
       , theirVersion :: Maybe ProtocolVersion
       }
+  | ClusterIDMismatch
+      { clusterPeers :: Text
+      }
   deriving stock (Generic, Eq, Show)
   deriving anyclass (ToJSON)
 
diff --git a/hydra-node/src/Hydra/Network/Etcd.hs b/hydra-node/src/Hydra/Network/Etcd.hs
@@ -56,16 +56,18 @@ import Control.Concurrent.Class.MonadSTM (
   writeTVar,
  )
 import Control.Exception (IOException)
-import Control.Lens ((^.), (^..))
+import Control.Lens ((^.), (^..), (^?))
 import Data.Aeson (decodeFileStrict', encodeFile)
 import Data.Aeson qualified as Aeson
+import Data.Aeson.Lens qualified as Aeson
 import Data.Aeson.Types (Value)
 import Data.Bits ((.|.))
 import Data.ByteString qualified as BS
 import Data.ByteString.Char8 qualified as BS8
 import Data.List ((\\))
 import Data.List qualified as List
 import Data.Map qualified as Map
+import Data.Text qualified as T
 import Hydra.Logging (Tracer, traceWith)
 import Hydra.Network (
   Connectivity (..),
@@ -138,12 +140,12 @@ withEtcdNetwork tracer protocolVersion config callback action = do
   envVars <- Map.fromList <$> getEnvironment
   withProcessInterrupt (etcdCmd etcdBinPath envVars) $ \p -> do
     race_ (waitExitCode p >>= \ec -> fail $ "Sub-process etcd exited with: " <> show ec) $ do
-      race_ (traceStderr p) $ do
+      race_ (traceStderr p callback) $ do
         -- XXX: cleanup reconnecting through policy if other threads fail
         doneVar <- newTVarIO False
         -- NOTE: The connection to the server is set up asynchronously; the
         -- first rpc call will block until the connection has been established.
-        withConnection (connParams doneVar) grpcServer $ \conn ->
+        withConnection (connParams doneVar) grpcServer $ \conn -> do
           -- REVIEW: checkVersion blocks if used on main thread - why?
           withAsync (checkVersion tracer conn protocolVersion callback) $ \_ -> do
             race_ (pollConnectivity tracer conn advertise callback) $
@@ -188,12 +190,20 @@ withEtcdNetwork tracer protocolVersion config callback action = do
   -- be used by default still.
   clientPort = 2379 + port listen - 5001
 
-  traceStderr p =
+  traceStderr p NetworkCallback{onConnectivity} =
     forever $ do
       bs <- BS.hGetLine (getStderr p)
       case Aeson.eitherDecodeStrict bs of
         Left err -> traceWith tracer FailedToDecodeLog{log = decodeUtf8 bs, reason = show err}
-        Right v -> traceWith tracer $ EtcdLog{etcd = v}
+        Right v -> do
+          let expectedClusterMismatch = do
+                level' <- bs ^? Aeson.key "level" . Aeson.nonNull
+                msg' <- bs ^? Aeson.key "msg" . Aeson.nonNull
+                pure (level', msg')
+          case expectedClusterMismatch of
+            Just (Aeson.String "error", Aeson.String "request sent was ignored due to cluster ID mismatch") ->
+              onConnectivity ClusterIDMismatch{clusterPeers = T.pack clusterPeers}
+            _ -> traceWith tracer $ EtcdLog{etcd = v}
 
   -- XXX: Could use TLS to secure peer connections
   -- XXX: Could use discovery to simplify configuration
diff --git a/hydra-node/src/Hydra/Node.hs b/hydra-node/src/Hydra/Node.hs
@@ -19,6 +19,7 @@ import Control.Concurrent.Class.MonadSTM (
   writeTVar,
  )
 import Control.Monad.Trans.Writer (execWriter, tell)
+import Data.Text (pack)
 import Hydra.API.ClientInput (ClientInput)
 import Hydra.API.Server (Server, sendMessage)
 import Hydra.Cardano.Api (
@@ -51,7 +52,7 @@ import Hydra.HeadLogic.State (getHeadParameters)
 import Hydra.HeadLogic.StateEvent (StateEvent (..))
 import Hydra.Ledger (Ledger)
 import Hydra.Logging (Tracer, traceWith)
-import Hydra.Network (Network (..), NetworkCallback (..))
+import Hydra.Network (Host (..), Network (..), NetworkCallback (..))
 import Hydra.Network.Authenticate (Authenticated (..))
 import Hydra.Network.Message (Message (..), NetworkEvent (..))
 import Hydra.Node.Environment (Environment (..))
@@ -78,6 +79,7 @@ initEnvironment options = do
       , participants
       , contestationPeriod
       , depositPeriod
+      , configuredPeers
       }
  where
   -- XXX: This is mostly a cardano-specific initialization step of loading
@@ -104,10 +106,20 @@ initEnvironment options = do
   loadParty p =
     Party <$> readFileTextEnvelopeThrow p
 
+  httpUrl (Host h p) = "http://" <> toString h <> ":" <> show p
+
+  configuredPeers =
+    pack
+      $ intercalate ","
+        . map (\h -> show h <> "=" <> httpUrl h)
+      $ (maybeToList advertise <> peers)
+
   RunOptions
     { hydraSigningKey
     , hydraVerificationKeys
     , chainConfig
+    , advertise
+    , peers
     } = options
 
 -- | Checks that command line options match a given 'HeadState'. This function
diff --git a/hydra-node/src/Hydra/Node/Environment.hs b/hydra-node/src/Hydra/Node/Environment.hs
@@ -20,6 +20,8 @@ data Environment = Environment
     participants :: [OnChainId]
   , contestationPeriod :: ContestationPeriod
   , depositPeriod :: DepositPeriod
+  , configuredPeers :: Text
+  -- ^ Configured peers for the network layer, used for comparison on etcd errors.
   }
   deriving stock (Generic, Show, Eq)
 
diff --git a/hydra-node/test/Hydra/BehaviorSpec.hs b/hydra-node/test/Hydra/BehaviorSpec.hs
@@ -1350,6 +1350,7 @@ createHydraNode tracer ledger chainState signingKey otherParties outputs message
       , contestationPeriod = cp
       , participants
       , depositPeriod = dp
+      , configuredPeers = ""
       }
   party = deriveParty signingKey
 
diff --git a/hydra-node/test/Hydra/HeadLogicSnapshotSpec.hs b/hydra-node/test/Hydra/HeadLogicSnapshotSpec.hs
@@ -56,6 +56,7 @@ spec = do
                 , contestationPeriod = defaultContestationPeriod
                 , depositPeriod = defaultDepositPeriod
                 , participants = deriveOnChainId <$> threeParties
+                , configuredPeers = ""
                 }
 
     let coordinatedHeadState =
@@ -201,6 +202,7 @@ prop_singleMemberHeadAlwaysSnapshotOnReqTx sn = monadicST $ do
             , contestationPeriod = defaultContestationPeriod
             , depositPeriod = defaultDepositPeriod
             , participants = [deriveOnChainId party]
+            , configuredPeers = ""
             }
     st =
       CoordinatedHeadState
diff --git a/hydra-node/test/Hydra/HeadLogicSpec.hs b/hydra-node/test/Hydra/HeadLogicSpec.hs
diff --git a/hydra-node/test/Hydra/NetworkSpec.hs b/hydra-node/test/Hydra/NetworkSpec.hs
diff --git a/hydra-node/test/Hydra/NodeSpec.hs b/hydra-node/test/Hydra/NodeSpec.hs
diff --git a/hydra-node/testlib/Test/Hydra/Node/Fixture.hs b/hydra-node/testlib/Test/Hydra/Node/Fixture.hs
