fallback and agent side stop sync

This commit adds a fallback in the case where all the agents in system
are draining. Rather than drop the request with error, we fallback to
the existing behavior i.e continue to the send the request to the agent
even if its draining.

As for the agent side issue, if the agent has sent the DRAIN signal to
the server, ideally it should stop doing the syncOnce with the server.
This mistakes the server the agent is back ready.

Signed-off-by: Imran Pochi <[email protected]>

Author: ipochi

pkg/agent/clientset.go

@@ -255,6 +255,14 @@ func (cs *ClientSet) sync() {
 }
 
 func (cs *ClientSet) connectOnce() error {
+	// Skip establishing new connections if draining
+	select {
+	case <-cs.drainCh:
+		klog.V(2).InfoS("Skipping connectOnce - agent is draining")
+		return nil
+	default:
+	}
+
 	serverCount := cs.determineServerCount()
 
 	// If not in syncForever mode, we only connect if we have fewer connections than the server count.

pkg/server/backend_manager.go

@@ -366,6 +366,8 @@ func (s *DefaultBackendStorage) GetRandomBackend() (*Backend, error) {
 		return nil, &ErrNotFound{}
 	}
 
+	var firstDrainingBackend *Backend
+
 	// Start at a random agent and check each agent in sequence
 	startIdx := s.random.Intn(len(s.agentIDs))
 	for i := 0; i < len(s.agentIDs); i++ {
@@ -380,9 +382,19 @@ func (s *DefaultBackendStorage) GetRandomBackend() (*Backend, error) {
 			klog.V(3).InfoS("Pick agent as backend", "agentID", agentID)
 			return backend, nil
 		}
+
+		// Keep track of first draining backend as fallback
+		if firstDrainingBackend == nil {
+			firstDrainingBackend = backend
+		}
+	}
+
+	// All agents are draining, use one as fallback
+	if firstDrainingBackend != nil {
+		agentID := firstDrainingBackend.id
+		klog.V(2).InfoS("No non-draining backends available, using draining backend as fallback", "agentID", agentID)
+		return firstDrainingBackend, nil
 	}
 
-	// All agents are draining
-	klog.V(2).InfoS("No non-draining backends available")
 	return nil, &ErrNotFound{}
 }

pkg/server/desthost_backend_manager.go

@@ -79,14 +79,25 @@ func (dibm *DestHostBackendManager) Backend(ctx context.Context) (*Backend, erro
 	if destHost != "" {
 		bes, exist := dibm.backends[destHost]
 		if exist && len(bes) > 0 {
+			var firstDrainingBackend *Backend
+
 			// Find a non-draining backend for this destination host
 			for _, backend := range bes {
 				if !backend.IsDraining() {
 					klog.V(5).InfoS("Get the backend through the DestHostBackendManager", "destHost", destHost)
 					return backend, nil
 				}
+				// Keep track of first draining backend as fallback
+				if firstDrainingBackend == nil {
+					firstDrainingBackend = backend
+				}
+			}
+
+			// All backends for this destination are draining, use one as fallback
+			if firstDrainingBackend != nil {
+				klog.V(4).InfoS("All backends for destination host are draining, using one as fallback", "destHost", destHost)
+				return firstDrainingBackend, nil
 			}
-			klog.V(4).InfoS("All backends for destination host are draining", "destHost", destHost)
 		}
 	}
 	return nil, &ErrNotFound{}