transition to oos from oos for load balanced multinode rpc (#66)

* transition to oos from oos

* review

Author: yashnevatia

multinode/node_fsm.go

@@ -239,7 +239,7 @@ func (n *node[CHAIN_ID, HEAD, RPC]) transitionToOutOfSync(fn func()) {
 		return
 	}
 	switch n.state {
-	case nodeStateAlive:
+	case nodeStateAlive, nodeStateOutOfSync:
 		n.rpc.Close()
 		n.state = nodeStateOutOfSync
 	default:

multinode/node_fsm_test.go

@@ -49,7 +49,7 @@ func TestUnit_Node_StateTransitions(t *testing.T) {
 	})
 	t.Run("transitionToOutOfSync", func(t *testing.T) {
 		const destinationState = nodeStateOutOfSync
-		allowedStates := []nodeState{nodeStateAlive}
+		allowedStates := []nodeState{nodeStateAlive, nodeStateOutOfSync}
 		rpc := newMockRPCClient[ID, Head](t)
 		rpc.On("Close")
 		testTransition(t, rpc, testNode.transitionToOutOfSync, destinationState, allowedStates...)

multinode/node_lifecycle.go

@@ -461,7 +461,8 @@ func (n *node[CHAIN_ID, HEAD, RPC]) outOfSyncLoop(syncIssues syncStatus) {
 			if n.poolInfoProvider != nil {
 				if l, _ := n.poolInfoProvider.LatestChainInfo(); l < 1 {
 					if n.isLoadBalancedRPC {
-						n.declareUnreachable()
+						// in case all rpcs behind a load balanced rpc are out of sync, we need to declare out of sync to prevent false transition to alive
+						n.declareOutOfSync(syncIssues)
 						return
 					}
 					lggr.Criticalw("RPC endpoint is still out of sync, but there are no other available nodes. This RPC node will be forcibly moved back into the live pool in a degraded state", "syncIssues", syncIssues)

multinode/node_lifecycle_test.go

@@ -227,7 +227,9 @@ func TestUnit_NodeLifecycle_aliveLoop(t *testing.T) {
 		rpc.On("ClientVersion", mock.Anything).Return("", pollError)
 		node.declareAlive()
 		tests.AssertLogEventually(t, observedLogs, fmt.Sprintf("RPC endpoint failed to respond to %d consecutive polls", pollFailureThreshold))
-		assert.Equal(t, nodeStateUnreachable, node.State())
+		tests.AssertEventually(t, func() bool {
+			return node.State() == nodeStateUnreachable
+		})
 	})
 	t.Run("when behind more than SyncThreshold, transitions to out of sync", func(t *testing.T) {
 		t.Parallel()