Remove routes of ex gw pods in terminating or not ready state

Signed-off-by: arkadeepsen <[email protected]>

Author: arkadeepsen

go-controller/pkg/ovn/controller/apbroute/external_controller.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
+	v1pod "k8s.io/kubernetes/pkg/api/v1/pod"
 
 	adminpolicybasedrouteapi "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1"
 	adminpolicybasedrouteinformer "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/crd/adminpolicybasedroute/v1/apis/informers/externalversions/adminpolicybasedroute/v1"
@@ -565,10 +566,14 @@ func (m *externalPolicyManager) onPodUpdate(oldObj, newObj interface{}) {
 		utilruntime.HandleError(errors.New("invalid Pod provided to onPodUpdate()"))
 		return
 	}
-	// if labels AND assigned Pod IPs AND the multus network status annotations are the same, skip processing changes to the pod.
+	// if labels AND assigned Pod IPs AND the multus network status annotations AND
+	// pod PodReady condition AND deletion timestamp (PodTerminating) are
+	// the same, skip processing changes to the pod.
 	if reflect.DeepEqual(o.Labels, n.Labels) &&
 		reflect.DeepEqual(o.Status.PodIPs, n.Status.PodIPs) &&
-		reflect.DeepEqual(o.Annotations[nettypes.NetworkStatusAnnot], n.Annotations[nettypes.NetworkStatusAnnot]) {
+		reflect.DeepEqual(o.Annotations[nettypes.NetworkStatusAnnot], n.Annotations[nettypes.NetworkStatusAnnot]) &&
+		reflect.DeepEqual(v1pod.GetPodReadyCondition(o.Status), v1pod.GetPodReadyCondition(n.Status)) &&
+		reflect.DeepEqual(o.DeletionTimestamp, n.DeletionTimestamp) {
 		return
 	}
 	m.podQueue.Add(n)

go-controller/pkg/ovn/controller/apbroute/external_controller_pod.go

@@ -11,7 +11,10 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
+	v1pod "k8s.io/kubernetes/pkg/api/v1/pod"
 	utilnet "k8s.io/utils/net"
+
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 )
 
 func (m *externalPolicyManager) syncPod(pod *corev1.Pod, routeQueue workqueue.TypedRateLimitingInterface[string]) error {
@@ -28,6 +31,13 @@ func (m *externalPolicyManager) syncPod(pod *corev1.Pod, routeQueue workqueue.Ty
 }
 
 func getExGwPodIPs(gatewayPod *corev1.Pod, networkName string) (sets.Set[string], error) {
+	// If an external gateway pod is in terminating or not ready state then don't return the
+	// IPs for the external gateway pod
+	if util.PodTerminating(gatewayPod) || !v1pod.IsPodReadyConditionTrue(gatewayPod.Status) {
+		klog.Warningf("External gateway pod cannot serve traffic; it's in terminating or not ready state: %s/%s", gatewayPod.Namespace, gatewayPod.Name)
+		return nil, nil
+	}
+
 	if networkName != "" {
 		return getMultusIPsFromNetworkName(gatewayPod, networkName)
 	}

go-controller/pkg/ovn/egressgw.go

@@ -15,6 +15,7 @@ import (
 	ktypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
+	v1pod "k8s.io/kubernetes/pkg/api/v1/pod"
 	utilnet "k8s.io/utils/net"
 
 	libovsdbclient "github.com/ovn-kubernetes/libovsdb/client"
@@ -49,6 +50,13 @@ func (oc *DefaultNetworkController) addPodExternalGW(pod *corev1.Pod) error {
 
 	klog.Infof("External gateway pod: %s, detected for namespace(s) %s", pod.Name, podRoutingNamespaceAnno)
 
+	// If an external gateway pod is in terminating or not ready state then don't add the
+	// routes for the external gateway pod
+	if util.PodTerminating(pod) || !v1pod.IsPodReadyConditionTrue(pod.Status) {
+		klog.Warningf("External gateway pod cannot serve traffic; it's in terminating or not ready state: %s/%s", pod.Namespace, pod.Name)
+		return nil
+	}
+
 	foundGws, err := getExGwPodIPs(pod)
 	if err != nil {
 		klog.Errorf("Error getting exgw IPs for pod: %s, error: %v", pod.Name, err)

go-controller/pkg/ovn/ovn.go

@@ -16,6 +16,7 @@ import (
 	listers "k8s.io/client-go/listers/core/v1"
 	ref "k8s.io/client-go/tools/reference"
 	"k8s.io/klog/v2"
+	v1pod "k8s.io/kubernetes/pkg/api/v1/pod"
 
 	libovsdbclient "github.com/ovn-kubernetes/libovsdb/client"
 
@@ -117,6 +118,10 @@ func networkStatusAnnotationsChanged(oldPod, newPod *corev1.Pod) bool {
 	return oldPod.Annotations[nettypes.NetworkStatusAnnot] != newPod.Annotations[nettypes.NetworkStatusAnnot]
 }
 
+func podBecameReady(oldPod, newPod *corev1.Pod) bool {
+	return !v1pod.IsPodReadyConditionTrue(oldPod.Status) && v1pod.IsPodReadyConditionTrue(newPod.Status)
+}
+
 // ensurePod tries to set up a pod. It returns nil on success and error on failure; failure
 // indicates the pod set up should be retried later.
 func (oc *DefaultNetworkController) ensurePod(oldPod, pod *corev1.Pod, addPort bool) error {
@@ -131,6 +136,14 @@ func (oc *DefaultNetworkController) ensurePod(oldPod, pod *corev1.Pod, addPort b
 		return oc.ensureRemotePodIP(oldPod, pod, addPort)
 	}
 
+	// If an external gateway pod is in terminating or not ready state then remove the
+	// routes for the external gateway pod
+	if util.PodTerminating(pod) || !v1pod.IsPodReadyConditionTrue(pod.Status) {
+		if err := oc.deletePodExternalGW(pod); err != nil {
+			return fmt.Errorf("ensurePod failed %s/%s: %w", pod.Namespace, pod.Name, err)
+		}
+	}
+
 	if oc.isPodScheduledinLocalZone(pod) {
 		klog.V(5).Infof("Ensuring zone local for Pod %s/%s in node %s", pod.Namespace, pod.Name, pod.Spec.NodeName)
 		return oc.ensureLocalZonePod(oldPod, pod, addPort)
@@ -170,7 +183,7 @@ func (oc *DefaultNetworkController) ensureLocalZonePod(oldPod, pod *corev1.Pod,
 		}
 	} else {
 		// either pod is host-networked or its an update for a normal pod (addPort=false case)
-		if oldPod == nil || exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod) {
+		if oldPod == nil || exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod) || podBecameReady(oldPod, pod) {
 			if err := oc.addPodExternalGW(pod); err != nil {
 				return fmt.Errorf("addPodExternalGW failed for %s/%s: %w", pod.Namespace, pod.Name, err)
 			}
@@ -237,7 +250,7 @@ func (oc *DefaultNetworkController) ensureRemoteZonePod(oldPod, pod *corev1.Pod,
 	}
 
 	// either pod is host-networked or its an update for a normal pod (addPort=false case)
-	if oldPod == nil || exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod) {
+	if oldPod == nil || exGatewayAnnotationsChanged(oldPod, pod) || networkStatusAnnotationsChanged(oldPod, pod) || podBecameReady(oldPod, pod) {
 		// check if this remote pod is serving as an external GW. If so add the routes in the namespace
 		// associated with this remote pod
 		if err := oc.addPodExternalGW(pod); err != nil {