Add more useful information on cluster misconfiguration

v0d1ch · v0d1ch · commit 766e84f50c78 · 2025-06-04T13:40:46.000+02:00
Also provide e2e test to assert we see what we expect in the logs

Signed-off-by: Sasha Bogicevic &lt;sasha.bogicevic@iohk.io&gt;
diff --git a/docs/docs/known-issues.md b/docs/docs/known-issues.md
@@ -28,7 +28,10 @@ The network topology needs to be statically configured and match across all `hyd
 Known errors are:
 
  - `cluster ID mismatch` - the cluster was initiated with a different list of `--peer`s
-   - check configuration with other participants
+   - check configuration with other participants. There should be a corresponding log entry `NetworkClusterIDMismatch` with the information on:
+      - `clusterPeers` - loaded peers info received from `etcd` cluster.
+      - `configuredPeers` - peers info coming from `hydra-node` arguments.
+      - `reportingHost` - host that received the error.
 
  - `member ... has already been bootstrapped` - missing information in `<persistence-dir>/etcd`
    - need to bootstrap new cluster or manual workarounds, see also https://etcd.io/docs/v3.5/op-guide/failures/
diff --git a/hydra-cluster/src/Hydra/Cluster/Scenarios.hs b/hydra-cluster/src/Hydra/Cluster/Scenarios.hs
@@ -1172,6 +1172,35 @@ initWithWrongKeys workDir tracer node@RunningNode{nodeSocket} hydraScriptsTxId =
 
       participants `shouldMatchList` expectedParticipants
 
+startWithWrongPeers :: FilePath -> Tracer IO EndToEndLog -> RunningNode -> [TxId] -> IO ()
+startWithWrongPeers workDir tracer node@RunningNode{nodeSocket} hydraScriptsTxId = do
+  (aliceCardanoVk, _) <- keysFor Alice
+
+  let contestationPeriod = 2
+  aliceChainConfig <- chainConfigFor Alice workDir nodeSocket hydraScriptsTxId [Carol] contestationPeriod
+  bobChainConfig <- chainConfigFor Bob workDir nodeSocket hydraScriptsTxId [Alice] contestationPeriod
+
+  let hydraTracer = contramap FromHydraNode tracer
+  withHydraNode hydraTracer aliceChainConfig workDir 3 aliceSk [bobVk] [3, 4] $ \n1 -> do
+    -- NOTE: here we deliberately use the wrong peer list for Bob, which should be visible
+    -- in network message sent to Alice.
+    withHydraNode hydraTracer bobChainConfig workDir 4 bobSk [aliceVk] [4] $ \_ -> do
+      seedFromFaucet_ node aliceCardanoVk 100_000_000 (contramap FromFaucet tracer)
+
+      (clusterPeers, reportingHost, configuredPeers) <- waitMatch 20 n1 $ \v -> do
+        guard $ v ^? key "tag" == Just (Aeson.String "NetworkClusterIDMismatch")
+        clusterPeers <- v ^? key "clusterPeers" . _String
+        reportingHost <- v ^? key "reportingHost" . _String
+        configuredPeers <- v ^? key "misconfiguredPeers" . _String
+        pure (clusterPeers, reportingHost, configuredPeers)
+
+      when (clusterPeers == configuredPeers) $
+        failure "Expected clusterPeers and configuredPeers to be different"
+      -- NOTE: reporting host is the host receiving the error, which is Alice
+      reportingHost `shouldBe` "http://0.0.0.0:5003"
+      clusterPeers `shouldBe` "0.0.0.0:5003=http://0.0.0.0:5003,0.0.0.0:5004=http://0.0.0.0:5004"
+      configuredPeers `shouldBe` "0.0.0.0:5004=http://0.0.0.0:5004"
+
 -- | Open a a two participant head and incrementally commit to it.
 canCommit :: Tracer IO EndToEndLog -> FilePath -> RunningNode -> [TxId] -> IO ()
 canCommit tracer workDir node hydraScriptsTxId =
diff --git a/hydra-cluster/test/Test/EndToEndSpec.hs b/hydra-cluster/test/Test/EndToEndSpec.hs
@@ -75,6 +75,7 @@ import Hydra.Cluster.Scenarios (
   singlePartyHeadFullLifeCycle,
   singlePartyUsesScriptOnL2,
   singlePartyUsesWithdrawZeroTrick,
+  startWithWrongPeers,
   threeNodesNoErrorsOnOpen,
   threeNodesWithMirrorParty,
  )
@@ -566,6 +567,13 @@ spec = around (showLogsOnFailure "EndToEndSpec") $ do
               publishHydraScriptsAs node Faucet
                 >>= initWithWrongKeys tmpDir tracer node
 
+      it "cluster id mismatch provides useful info in the logs" $ \tracer ->
+        failAfter 60 $
+          withClusterTempDir $ \tmpDir -> do
+            withCardanoNodeDevnet (contramap FromCardanoNode tracer) tmpDir $ \node -> do
+              publishHydraScriptsAs node Faucet
+                >>= startWithWrongPeers tmpDir tracer node
+
       it "bob cannot abort alice's head" $ \tracer -> do
         failAfter 60 $
           withClusterTempDir $ \tmpDir -> do
diff --git a/hydra-node/golden/ServerOutput/NetworkClusterIDMismatch.json b/hydra-node/golden/ServerOutput/NetworkClusterIDMismatch.json
@@ -1,8 +1,9 @@
 {
     "samples": [
         {
-            "localClusterID": "",
-            "remotePeerClusterID": "",
+            "clusterPeers": "",
+            "misconfiguredPeers": "`",
+            "reportingHost": "",
             "tag": "NetworkClusterIDMismatch"
         }
     ],
diff --git a/hydra-node/golden/StateChanged/NetworkClusterIDMismatch.json b/hydra-node/golden/StateChanged/NetworkClusterIDMismatch.json
@@ -1,13 +1,9 @@
 {
     "samples": [
         {
-            "configuredParties": [
-                {
-                    "vkey": "d05f801e08648c87a1182768655803d4261ae2545573cd3a58b837f1541691de"
-                }
-            ],
-            "localClusterID": "",
-            "remotePeerClusterID": "",
+            "clusterPeers": "",
+            "misconfiguredPeers": "$",
+            "reportingHost": "",
             "tag": "NetworkClusterIDMismatch"
         }
     ],
diff --git a/hydra-node/json-schemas/api.yaml b/hydra-node/json-schemas/api.yaml
@@ -1004,14 +1004,12 @@ components:
         tag:
           type: string
           enum: ["NetworkClusterIDMismatch"]
-        localClusterID:
+        clusterPeers:
           type: string
-        remotePeerClusterID:
+        reportingHost:
+          type: string
+        misconfiguredPeers:
           type: string
-        configuredParties:
-          type: array
-          items:
-            $ref: "api.yaml#/components/schemas/Party"
         seq:
           $ref: "api.yaml#/components/schemas/SequenceNumber"
         timestamp:
diff --git a/hydra-node/src/Hydra/API/ServerOutput.hs b/hydra-node/src/Hydra/API/ServerOutput.hs
@@ -143,8 +143,9 @@ data ServerOutput tx
       , theirVersion :: Maybe ProtocolVersion
       }
   | NetworkClusterIDMismatch
-      { localClusterID :: Text
-      , remotePeerClusterID :: Text
+      { clusterPeers :: Text
+      , reportingHost :: Text
+      , misconfiguredPeers :: Text
       }
   | PeerConnected {peer :: Host}
   | PeerDisconnected {peer :: Host}
diff --git a/hydra-node/src/Hydra/HeadLogic.hs b/hydra-node/src/Hydra/HeadLogic.hs
@@ -104,16 +104,16 @@ import Hydra.Tx.Party (Party (vkey))
 import Hydra.Tx.Snapshot (ConfirmedSnapshot (..), Snapshot (..), SnapshotNumber, SnapshotVersion, getSnapshot)
 
 
-onConnectionEvent :: [Party] -> Network.Connectivity -> Outcome tx
-onConnectionEvent configuredParties = \case
+onConnectionEvent :: Text -> Network.Connectivity -> Outcome tx
+onConnectionEvent misconfiguredPeers = \case
   Network.NetworkConnected ->
     newState NetworkConnected
   Network.NetworkDisconnected ->
     newState NetworkDisconnected
   Network.VersionMismatch{ourVersion, theirVersion} ->
     newState NetworkVersionMismatch{ourVersion, theirVersion}
-  Network.ClusterIDMismatch{localClusterID, remotePeerClusterID} ->
-    newState NetworkClusterIDMismatch{localClusterID, remotePeerClusterID, configuredParties}
+  Network.ClusterIDMismatch{clusterPeers, reportingHost} ->
+    newState NetworkClusterIDMismatch{clusterPeers, reportingHost, misconfiguredPeers}
   Network.PeerConnected{peer} ->
     newState PeerConnected{peer}
   Network.PeerDisconnected{peer} ->
@@ -1316,7 +1316,7 @@ update ::
   Outcome tx
 update env ledger st ev = case (st, ev) of
   (_, NetworkInput _ (ConnectivityEvent conn)) ->
-    onConnectionEvent env.otherParties conn
+    onConnectionEvent env.configuredPeers conn
   (Idle _, ClientInput Init) ->
     onIdleClientInit env
   (Idle _, ChainInput Observation{observedTx = OnInitTx{headId, headSeed, headParameters, participants}, newChainState}) ->
diff --git a/hydra-node/src/Hydra/HeadLogic/Outcome.hs b/hydra-node/src/Hydra/HeadLogic/Outcome.hs
@@ -61,9 +61,9 @@ data StateChanged tx
       , theirVersion :: Maybe ProtocolVersion
       }
   | NetworkClusterIDMismatch
-      { localClusterID :: Text
-      , remotePeerClusterID :: Text
-      , configuredParties :: [Party]
+      { clusterPeers :: Text
+      , reportingHost :: Text
+      , misconfiguredPeers :: Text
       }
   | HeadInitialized
       { parameters :: HeadParameters
diff --git a/hydra-node/src/Hydra/Network.hs b/hydra-node/src/Hydra/Network.hs
@@ -186,8 +186,8 @@ data Connectivity
       , theirVersion :: Maybe ProtocolVersion
       }
   | ClusterIDMismatch
-      { localClusterID :: Text
-      , remotePeerClusterID :: Text
+      { clusterPeers :: Text
+      , reportingHost :: Text
       }
   deriving stock (Generic, Eq, Show)
   deriving anyclass (ToJSON)
diff --git a/hydra-node/src/Hydra/Network/Etcd.hs b/hydra-node/src/Hydra/Network/Etcd.hs
@@ -67,6 +67,7 @@ import Data.ByteString.Char8 qualified as BS8
 import Data.List ((\\))
 import Data.List qualified as List
 import Data.Map qualified as Map
+import Data.Text qualified as T
 import Hydra.Logging (Tracer, traceWith)
 import Hydra.Network (
   Connectivity (..),
@@ -198,12 +199,10 @@ withEtcdNetwork tracer protocolVersion config callback action = do
           let expectedClusterMismatch = do
                 level' <- bs ^? Aeson.key "level" . Aeson.nonNull
                 msg' <- bs ^? Aeson.key "msg" . Aeson.nonNull
-                localClusterId <- bs ^? Aeson.key "local-member-cluster-id" . Aeson.nonNull
-                remoteClusterId <- bs ^? Aeson.key "remote-peer-cluster-id" . Aeson.nonNull
-                pure (level', msg', localClusterId, remoteClusterId)
+                pure (level', msg')
           case expectedClusterMismatch of
-            Just (Aeson.String "error", Aeson.String "request sent was ignored due to cluster ID mismatch", Aeson.String localClusterID, Aeson.String remotePeerClusterID) ->
-              onConnectivity ClusterIDMismatch{localClusterID, remotePeerClusterID}
+            Just (Aeson.String "error", Aeson.String "request sent was ignored due to cluster ID mismatch") ->
+              onConnectivity ClusterIDMismatch{clusterPeers = T.pack clusterPeers, reportingHost = T.pack $ httpUrl advertise}
             _ -> traceWith tracer $ EtcdLog{etcd = v}
 
   -- XXX: Could use TLS to secure peer connections
diff --git a/hydra-node/src/Hydra/Node.hs b/hydra-node/src/Hydra/Node.hs
@@ -19,6 +19,7 @@ import Control.Concurrent.Class.MonadSTM (
   writeTVar,
  )
 import Control.Monad.Trans.Writer (execWriter, tell)
+import Data.Text (pack)
 import Hydra.API.ClientInput (ClientInput)
 import Hydra.API.Server (Server, sendMessage)
 import Hydra.Cardano.Api (
@@ -51,7 +52,7 @@ import Hydra.HeadLogic.State (getHeadParameters)
 import Hydra.HeadLogic.StateEvent (StateEvent (..))
 import Hydra.Ledger (Ledger)
 import Hydra.Logging (Tracer, traceWith)
-import Hydra.Network (Network (..), NetworkCallback (..))
+import Hydra.Network (Host (..), Network (..), NetworkCallback (..))
 import Hydra.Network.Authenticate (Authenticated (..))
 import Hydra.Network.Message (Message (..), NetworkEvent (..))
 import Hydra.Node.Environment (Environment (..))
@@ -78,6 +79,7 @@ initEnvironment options = do
       , participants
       , contestationPeriod
       , depositPeriod
+      , configuredPeers
       }
  where
   -- XXX: This is mostly a cardano-specific initialization step of loading
@@ -104,10 +106,20 @@ initEnvironment options = do
   loadParty p =
     Party <$> readFileTextEnvelopeThrow p
 
+  httpUrl (Host h p) = "http://" <> toString h <> ":" <> show p
+
+  configuredPeers =
+    pack
+      $ intercalate ","
+        . map (\h -> show h <> "=" <> httpUrl h)
+      $ (maybeToList advertise <> peers)
+
   RunOptions
     { hydraSigningKey
     , hydraVerificationKeys
     , chainConfig
+    , advertise
+    , peers
     } = options
 
 -- | Checks that command line options match a given 'HeadState'. This function
diff --git a/hydra-node/src/Hydra/Node/Environment.hs b/hydra-node/src/Hydra/Node/Environment.hs
@@ -20,6 +20,8 @@ data Environment = Environment
     participants :: [OnChainId]
   , contestationPeriod :: ContestationPeriod
   , depositPeriod :: DepositPeriod
+  , configuredPeers :: Text
+  -- ^ Configured peers for the network layer, used for comparison on etcd errors.
   }
   deriving stock (Generic, Show, Eq)
 
diff --git a/hydra-node/test/Hydra/BehaviorSpec.hs b/hydra-node/test/Hydra/BehaviorSpec.hs
@@ -1350,6 +1350,7 @@ createHydraNode tracer ledger chainState signingKey otherParties outputs message
       , contestationPeriod = cp
       , participants
       , depositPeriod = dp
+      , configuredPeers = ""
       }
   party = deriveParty signingKey
 
diff --git a/hydra-node/test/Hydra/HeadLogicSnapshotSpec.hs b/hydra-node/test/Hydra/HeadLogicSnapshotSpec.hs
@@ -56,6 +56,7 @@ spec = do
                 , contestationPeriod = defaultContestationPeriod
                 , depositPeriod = defaultDepositPeriod
                 , participants = deriveOnChainId <$> threeParties
+                , configuredPeers = ""
                 }
 
     let coordinatedHeadState =
@@ -201,6 +202,7 @@ prop_singleMemberHeadAlwaysSnapshotOnReqTx sn = monadicST $ do
             , contestationPeriod = defaultContestationPeriod
             , depositPeriod = defaultDepositPeriod
             , participants = [deriveOnChainId party]
+            , configuredPeers = ""
             }
     st =
       CoordinatedHeadState
diff --git a/hydra-node/test/Hydra/HeadLogicSpec.hs b/hydra-node/test/Hydra/HeadLogicSpec.hs
@@ -83,6 +83,7 @@ spec =
             , contestationPeriod = defaultContestationPeriod
             , depositPeriod = defaultDepositPeriod
             , participants = deriveOnChainId <$> threeParties
+            , configuredPeers = ""
             }
         aliceEnv =
           Environment
@@ -92,6 +93,7 @@ spec =
             , contestationPeriod = defaultContestationPeriod
             , depositPeriod = defaultDepositPeriod
             , participants = deriveOnChainId <$> threeParties
+            , configuredPeers = ""
             }
 
     describe "Coordinated Head Protocol" $ do
diff --git a/hydra-node/test/Hydra/NodeSpec.hs b/hydra-node/test/Hydra/NodeSpec.hs
@@ -283,6 +283,7 @@ spec = parallel $ do
             , contestationPeriod = defaultContestationPeriod
             , depositPeriod = defaultDepositPeriod
             , participants = error "should not be recorded in head state"
+            , configuredPeers = ""
             }
         headState = inInitialState [alice, bob]
 
@@ -444,6 +445,7 @@ testHydraNode tracer signingKey otherParties contestationPeriod inputs = do
       , contestationPeriod
       , depositPeriod = defaultDepositPeriod
       , participants
+      , configuredPeers = ""
       }
 
   party = deriveParty signingKey
diff --git a/hydra-node/testlib/Test/Hydra/Node/Fixture.hs b/hydra-node/testlib/Test/Hydra/Node/Fixture.hs
@@ -49,4 +49,5 @@ testEnvironment =
     , contestationPeriod = cperiod
     , depositPeriod = DepositPeriod 20
     , participants = deriveOnChainId <$> [alice, bob, carol]
+    , configuredPeers = ""
     }

Original file line number	Diff line number	Diff line change
`@@ -1,8 +1,9 @@`
`1`	`1`	`{`
`2`	`2`	`"samples": [`
`3`	`3`	`{`
`4`		`- "localClusterID": "",`
`5`		`- "remotePeerClusterID": "",`
	`4`	`+ "clusterPeers": "",`
	`5`	+ "misconfiguredPeers": "`",
	`6`	`+ "reportingHost": "",`
`6`	`7`	`"tag": "NetworkClusterIDMismatch"`
`7`	`8`	`}`
`8`	`9`	`],`
Original file line number	Diff line number	Diff line change
`@@ -1,13 +1,9 @@`
`1`	`1`	`{`
`2`	`2`	`"samples": [`
`3`	`3`	`{`
`4`		`- "configuredParties": [`
`5`		`- {`
`6`		`- "vkey": "d05f801e08648c87a1182768655803d4261ae2545573cd3a58b837f1541691de"`
`7`		`- }`
`8`		`- ],`
`9`		`- "localClusterID": "",`
`10`		`- "remotePeerClusterID": "",`
	`4`	`+ "clusterPeers": "",`
	`5`	`+ "misconfiguredPeers": "$",`
	`6`	`+ "reportingHost": "",`
`11`	`7`	`"tag": "NetworkClusterIDMismatch"`
`12`	`8`	`}`
`13`	`9`	`],`
Original file line number	Diff line number	Diff line change
`@@ -143,8 +143,9 @@ data ServerOutput tx`
`143`	`143`	`, theirVersion :: Maybe ProtocolVersion`
`144`	`144`	`}`
`145`	`145`	`\| NetworkClusterIDMismatch`
`146`		`- { localClusterID :: Text`
`147`		`- , remotePeerClusterID :: Text`
	`146`	`+ { clusterPeers :: Text`
	`147`	`+ , reportingHost :: Text`
	`148`	`+ , misconfiguredPeers :: Text`
`148`	`149`	`}`
`149`	`150`	`\| PeerConnected {peer :: Host}`
`150`	`151`	`\| PeerDisconnected {peer :: Host}`
Original file line number	Diff line number	Diff line change
`@@ -61,9 +61,9 @@ data StateChanged tx`
`61`	`61`	`, theirVersion :: Maybe ProtocolVersion`
`62`	`62`	`}`
`63`	`63`	`\| NetworkClusterIDMismatch`
`64`		`- { localClusterID :: Text`
`65`		`- , remotePeerClusterID :: Text`
`66`		`- , configuredParties :: [Party]`
	`64`	`+ { clusterPeers :: Text`
	`65`	`+ , reportingHost :: Text`
	`66`	`+ , misconfiguredPeers :: Text`
`67`	`67`	`}`
`68`	`68`	`\| HeadInitialized`
`69`	`69`	`{ parameters :: HeadParameters`
Original file line number	Diff line number	Diff line change
`@@ -186,8 +186,8 @@ data Connectivity`
`186`	`186`	`, theirVersion :: Maybe ProtocolVersion`
`187`	`187`	`}`
`188`	`188`	`\| ClusterIDMismatch`
`189`		`- { localClusterID :: Text`
`190`		`- , remotePeerClusterID :: Text`
	`189`	`+ { clusterPeers :: Text`
	`190`	`+ , reportingHost :: Text`
`191`	`191`	`}`
`192`	`192`	`deriving stock (Generic, Eq, Show)`
`193`	`193`	`deriving anyclass (ToJSON)`