Skip to content

Commit d565fd8

Browse files
arkadeepsenarkadeepsen
committed
Add e2e tests for ex gw pods in terminating or not ready state
Signed-off-by: arkadeepsen <[email protected]>
1 parent d942a7d commit d565fd8

File tree

1 file changed

+186
-27
lines changed

1 file changed

+186
-27
lines changed

‎test/e2e/external_gateways.go‎

Lines changed: 186 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,16 @@ const (
4242
anyLink = "any"
4343
)
4444

45+
// GatewayRemovalType defines ways to remove pod as external gateway
46+
type GatewayRemovalType string
47+
48+
const (
49+
GatewayUpdate GatewayRemovalType = "GatewayUpdate"
50+
GatewayDelete GatewayRemovalType = "GatewayDelete"
51+
GatewayDeletionTimestamp GatewayRemovalType = "GatewayDeletionTimestamp"
52+
GatewayNotReady GatewayRemovalType = "GatewayNotReady"
53+
)
54+
4555
func getOverrideNetwork() (string, string, string) {
4656
// When the env variable is specified, we use a different docker network for
4757
// containers acting as external gateways.
@@ -875,10 +885,15 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
875885
ginkgo.Entry("IPV6 udp", &addressesv6, "udp"),
876886
ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp"))
877887

878-
ginkgo.DescribeTable("ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes", func(addresses *gatewayTestIPs, protocol string, deletePod bool) {
888+
ginkgo.DescribeTable("ExternalGWPod annotation: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes", func(addresses *gatewayTestIPs, protocol string, removalType GatewayRemovalType) {
879889
if addresses.srcPodIP == "" || addresses.nodeIP == "" {
880890
skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP)
881891
}
892+
893+
if removalType == GatewayNotReady {
894+
recreatePodWithReadinessProbe(f, gatewayPodName2, nodes.Items[1].Name, servingNamespace, sleepCommand, nil)
895+
}
896+
882897
ginkgo.By("Annotate the external gw pods to manage the src app pod namespace")
883898
for i, gwPod := range []string{gatewayPodName1, gatewayPodName2} {
884899
networkIPs := fmt.Sprintf("\"%s\"", addresses.gatewayIPs[i])
@@ -925,15 +940,9 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
925940
totalPodConnEntries := pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)
926941
gomega.Expect(totalPodConnEntries).To(gomega.Equal(6)) // total conntrack entries for this pod/protocol
927942

928-
if deletePod {
929-
ginkgo.By(fmt.Sprintf("Delete second external gateway pod %s from ns %s", gatewayPodName2, servingNamespace))
930-
err = f.ClientSet.CoreV1().Pods(servingNamespace).Delete(context.TODO(), gatewayPodName2, metav1.DeleteOptions{})
931-
framework.ExpectNoError(err, "Delete the gateway pod failed: %v", err)
932-
// give some time to handle pod delete event
933-
time.Sleep(5 * time.Second)
934-
} else {
935-
ginkgo.By("Remove second external gateway pod's routing-namespace annotation")
936-
annotatePodForGateway(gatewayPodName2, servingNamespace, "", addresses.gatewayIPs[1], false)
943+
cleanUpFn := handleGatewayPodRemoval(f, removalType, gatewayPodName2, servingNamespace, addresses.gatewayIPs[1], true)
944+
if cleanUpFn != nil {
945+
defer cleanUpFn()
937946
}
938947

939948
// ensure the conntrack deletion tracker annotation is updated
@@ -973,12 +982,20 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
973982
gomega.Expect(podConnEntriesWithMACLabelsSet).To(gomega.Equal(0)) // we don't have any remaining gateways left
974983
gomega.Expect(totalPodConnEntries).To(gomega.Equal(4)) // 6-2
975984
},
976-
ginkgo.Entry("IPV4 udp", &addressesv4, "udp", false),
977-
ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp", false),
978-
ginkgo.Entry("IPV6 udp", &addressesv6, "udp", false),
979-
ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp", false),
980-
ginkgo.Entry("IPV4 udp + pod delete", &addressesv4, "udp", true),
981-
ginkgo.Entry("IPV6 tcp + pod delete", &addressesv6, "tcp", true),
985+
ginkgo.Entry("IPV4 udp + pod annotation update", &addressesv4, "udp", GatewayUpdate),
986+
ginkgo.Entry("IPV4 tcp + pod annotation update", &addressesv4, "tcp", GatewayUpdate),
987+
ginkgo.Entry("IPV6 udp + pod annotation update", &addressesv6, "udp", GatewayUpdate),
988+
ginkgo.Entry("IPV6 tcp + pod annotation update", &addressesv6, "tcp", GatewayUpdate),
989+
ginkgo.Entry("IPV4 udp + pod delete", &addressesv4, "udp", GatewayDelete),
990+
ginkgo.Entry("IPV6 tcp + pod delete", &addressesv6, "tcp", GatewayDelete),
991+
ginkgo.Entry("IPV4 udp + pod deletion timestamp", &addressesv4, "udp", GatewayDeletionTimestamp),
992+
ginkgo.Entry("IPV4 tcp + pod deletion timestamp", &addressesv4, "tcp", GatewayDeletionTimestamp),
993+
ginkgo.Entry("IPV6 udp + pod deletion timestamp", &addressesv6, "udp", GatewayDeletionTimestamp),
994+
ginkgo.Entry("IPV6 tcp + pod deletion timestamp", &addressesv6, "tcp", GatewayDeletionTimestamp),
995+
ginkgo.Entry("IPV4 udp + pod not ready", &addressesv4, "udp", GatewayNotReady),
996+
ginkgo.Entry("IPV4 tcp + pod not ready", &addressesv4, "tcp", GatewayNotReady),
997+
ginkgo.Entry("IPV6 udp + pod not ready", &addressesv6, "udp", GatewayNotReady),
998+
ginkgo.Entry("IPV6 tcp + pod not ready", &addressesv6, "tcp", GatewayNotReady),
982999
)
9831000
})
9841001

@@ -1983,11 +2000,15 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
19832000
ginkgo.Entry("IPV6 udp", &addressesv6, "udp"),
19842001
ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp"))
19852002

1986-
ginkgo.DescribeTable("Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes", func(addresses *gatewayTestIPs, protocol string) {
2003+
ginkgo.DescribeTable("Dynamic Hop: Should validate conntrack entry deletion for TCP/UDP traffic via multiple external gateways a.k.a ECMP routes", func(addresses *gatewayTestIPs, protocol string, removalType GatewayRemovalType) {
19872004
if addresses.srcPodIP == "" || addresses.nodeIP == "" {
19882005
skipper.Skipf("Skipping as pod ip / node ip are not set pod ip %s node ip %s", addresses.srcPodIP, addresses.nodeIP)
19892006
}
19902007

2008+
if removalType == GatewayNotReady {
2009+
recreatePodWithReadinessProbe(f, gatewayPodName2, nodes.Items[1].Name, servingNamespace, sleepCommand, map[string]string{"name": gatewayPodName2, "gatewayPod": "true"})
2010+
}
2011+
19912012
for i, gwPod := range []string{gatewayPodName1, gatewayPodName2} {
19922013
annotateMultusNetworkStatusInPodGateway(gwPod, servingNamespace, []string{addresses.gatewayIPs[i], addresses.gatewayIPs[i]})
19932014
}
@@ -2026,10 +2047,10 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
20262047
}, time.Minute, 5).Should(gomega.Equal(podConnEntriesWithMACLabelsSet))
20272048
gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(totalPodConnEntries)) // total conntrack entries for this pod/protocol
20282049

2029-
ginkgo.By("Remove second external gateway pod's routing-namespace annotation")
2030-
p := getGatewayPod(f, servingNamespace, gatewayPodName2)
2031-
p.Labels = map[string]string{"name": gatewayPodName2}
2032-
updatePod(f, p)
2050+
cleanUpFn := handleGatewayPodRemoval(f, removalType, gatewayPodName2, servingNamespace, addresses.gatewayIPs[1], false)
2051+
if cleanUpFn != nil {
2052+
defer cleanUpFn()
2053+
}
20332054

20342055
ginkgo.By("Check if conntrack entries for ECMP routes are removed for the deleted external gateway if traffic is UDP")
20352056

@@ -2044,7 +2065,7 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
20442065
gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(totalPodConnEntries))
20452066

20462067
ginkgo.By("Remove first external gateway pod's routing-namespace annotation")
2047-
p = getGatewayPod(f, servingNamespace, gatewayPodName1)
2068+
p := getGatewayPod(f, servingNamespace, gatewayPodName1)
20482069
p.Labels = map[string]string{"name": gatewayPodName1}
20492070
updatePod(f, p)
20502071

@@ -2060,11 +2081,19 @@ var _ = ginkgo.Describe("External Gateway", feature.ExternalGateway, func() {
20602081
gomega.Expect(pokeConntrackEntries(nodeName, addresses.srcPodIP, protocol, nil)).To(gomega.Equal(totalPodConnEntries))
20612082
checkAPBExternalRouteStatus(defaultPolicyName)
20622083
},
2063-
ginkgo.Entry("IPV4 udp", &addressesv4, "udp"),
2064-
ginkgo.Entry("IPV4 tcp", &addressesv4, "tcp"),
2065-
ginkgo.Entry("IPV6 udp", &addressesv6, "udp"),
2066-
ginkgo.Entry("IPV6 tcp", &addressesv6, "tcp"))
2067-
2084+
ginkgo.Entry("IPV4 udp + pod annotation update", &addressesv4, "udp", GatewayUpdate),
2085+
ginkgo.Entry("IPV4 tcp + pod annotation update", &addressesv4, "tcp", GatewayUpdate),
2086+
ginkgo.Entry("IPV6 udp + pod annotation update", &addressesv6, "udp", GatewayUpdate),
2087+
ginkgo.Entry("IPV6 tcp + pod annotation update", &addressesv6, "tcp", GatewayUpdate),
2088+
ginkgo.Entry("IPV4 udp + pod deletion timestamp", &addressesv4, "udp", GatewayDeletionTimestamp),
2089+
ginkgo.Entry("IPV4 tcp + pod deletion timestamp", &addressesv4, "tcp", GatewayDeletionTimestamp),
2090+
ginkgo.Entry("IPV6 udp + pod deletion timestamp", &addressesv6, "udp", GatewayDeletionTimestamp),
2091+
ginkgo.Entry("IPV6 tcp + pod deletion timestamp", &addressesv6, "tcp", GatewayDeletionTimestamp),
2092+
ginkgo.Entry("IPV4 udp + pod not ready", &addressesv4, "udp", GatewayNotReady),
2093+
ginkgo.Entry("IPV4 tcp + pod not ready", &addressesv4, "tcp", GatewayNotReady),
2094+
ginkgo.Entry("IPV6 udp + pod not ready", &addressesv6, "udp", GatewayNotReady),
2095+
ginkgo.Entry("IPV6 tcp + pod not ready", &addressesv6, "tcp", GatewayNotReady),
2096+
)
20682097
})
20692098

20702099
// BFD Tests are dual of external gateway. The only difference is that they enable BFD on ovn and
@@ -3595,3 +3624,133 @@ func resetGatewayAnnotations(f *framework.Framework) {
35953624
annotation}...)
35963625
}
35973626
}
3627+
3628+
func setupPodWithReadinessProbe(f *framework.Framework, podName, nodeSelector, namespace string, command []string, labels map[string]string) (*corev1.Pod, error) {
3629+
// Handle bash -c commands specially to preserve argument structure
3630+
if len(command) >= 3 && command[0] == "bash" && command[1] == "-c" {
3631+
// Extract the script part and wrap it to preserve logic
3632+
script := strings.Join(command[2:], " ")
3633+
command = []string{"bash", "-c", "touch /tmp/ready && (" + script + ")"}
3634+
} else {
3635+
// For non-bash commands, preserve their structure
3636+
var quotedArgs []string
3637+
for _, arg := range command {
3638+
// Escape single quotes and wrap in single quotes
3639+
escaped := strings.ReplaceAll(arg, "'", "'\"'\"'")
3640+
quotedArgs = append(quotedArgs, "'"+escaped+"'")
3641+
}
3642+
command = []string{"bash", "-c", "touch /tmp/ready && " + strings.Join(quotedArgs, " ")}
3643+
}
3644+
return createPod(f, podName, nodeSelector, namespace, command, labels, func(p *corev1.Pod) {
3645+
p.Spec.Containers[0].ReadinessProbe = &corev1.Probe{
3646+
ProbeHandler: corev1.ProbeHandler{
3647+
Exec: &corev1.ExecAction{
3648+
Command: []string{"cat", "/tmp/ready"},
3649+
},
3650+
},
3651+
InitialDelaySeconds: 5,
3652+
PeriodSeconds: 5,
3653+
FailureThreshold: 1,
3654+
}
3655+
})
3656+
}
3657+
3658+
func recreatePodWithReadinessProbe(f *framework.Framework, podName, nodeSelector, namespace string, command []string, labels map[string]string) {
3659+
ginkgo.By(fmt.Sprintf("Delete second external gateway pod %s from ns %s", podName, namespace))
3660+
err := deletePodWithWaitByName(context.TODO(), f.ClientSet, podName, namespace)
3661+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Delete second external gateway pod %s from ns %s, failed: %v", podName, namespace, err))
3662+
3663+
ginkgo.By(fmt.Sprintf("Create second external gateway pod %s from ns %s with readiness probe", podName, namespace))
3664+
_, err = setupPodWithReadinessProbe(f, podName, nodeSelector, namespace, command, labels)
3665+
gomega.Expect(err).NotTo(gomega.HaveOccurred(), fmt.Sprintf("Create second external gateway pod %s from ns %s with readiness probe, failed: %v", podName, namespace, err))
3666+
gomega.Eventually(func() bool {
3667+
var p *corev1.Pod
3668+
p, err = f.ClientSet.CoreV1().Pods(namespace).Get(context.Background(), podName, metav1.GetOptions{})
3669+
if err != nil {
3670+
return false
3671+
}
3672+
for _, condition := range p.Status.Conditions {
3673+
if condition.Type == corev1.PodReady {
3674+
return condition.Status == corev1.ConditionTrue
3675+
}
3676+
}
3677+
return false
3678+
}).Should(gomega.Equal(true), fmt.Sprintf("Readiness probe for second external gateway pod %s from ns %s, failed: %v", podName, namespace, err))
3679+
}
3680+
3681+
func handleGatewayPodRemoval(f *framework.Framework, removalType GatewayRemovalType, gatewayPodName, servingNamespace, gatewayIP string, isAnnotated bool) func() {
3682+
var err error
3683+
switch removalType {
3684+
case GatewayDelete:
3685+
ginkgo.By(fmt.Sprintf("Delete second external gateway pod %s from ns %s", gatewayPodName, servingNamespace))
3686+
err := deletePodWithWaitByName(context.TODO(), f.ClientSet, gatewayPodName, servingNamespace)
3687+
framework.ExpectNoError(err, "Delete the gateway pod failed: %v", err)
3688+
return nil
3689+
case GatewayUpdate:
3690+
if isAnnotated {
3691+
ginkgo.By("Remove second external gateway pod's routing-namespace annotation")
3692+
annotatePodForGateway(gatewayPodName, servingNamespace, "", gatewayIP, false)
3693+
return nil
3694+
}
3695+
3696+
ginkgo.By("Updating external gateway pod labels")
3697+
p := getGatewayPod(f, servingNamespace, gatewayPodName)
3698+
p.Labels = map[string]string{"name": gatewayPodName}
3699+
updatePod(f, p)
3700+
return nil
3701+
case GatewayDeletionTimestamp:
3702+
ginkgo.By("Setting finalizer then deleting external gateway pod with grace period to set deletion timestamp")
3703+
p := getGatewayPod(f, servingNamespace, gatewayPodName)
3704+
p.Finalizers = append(p.Finalizers, "k8s.ovn.org/external-gw-pod-finalizer")
3705+
updatePod(f, p)
3706+
gomega.Eventually(func() bool {
3707+
p, err = f.ClientSet.CoreV1().Pods(servingNamespace).Get(context.Background(), gatewayPodName, metav1.GetOptions{})
3708+
if err != nil {
3709+
return false
3710+
}
3711+
return strings.Contains(strings.Join(p.GetFinalizers(), ","), "k8s.ovn.org/external-gw-pod-finalizer")
3712+
}).Should(gomega.Equal(true), fmt.Sprintf("Update second external gateway pod %s from ns %s with finalizer, failed: %v", gatewayPodName, servingNamespace, err))
3713+
3714+
p = getGatewayPod(f, servingNamespace, gatewayPodName)
3715+
err = e2epod.DeletePodWithGracePeriod(context.Background(), f.ClientSet, p, 1000)
3716+
framework.ExpectNoError(err, fmt.Sprintf("unable to delete pod with grace period: %s, err: %v", p.Name, err))
3717+
3718+
gomega.Eventually(func() bool {
3719+
p, err = f.ClientSet.CoreV1().Pods(servingNamespace).Get(context.Background(), gatewayPodName, metav1.GetOptions{})
3720+
if err != nil {
3721+
return false
3722+
}
3723+
return p.DeletionTimestamp != nil
3724+
}).Should(gomega.BeTrue(), fmt.Sprintf("Gateway pod %s in ns %s should have deletion timestamp, failed: %v", gatewayPodName, servingNamespace, err))
3725+
3726+
// return a function to remove the finalizer
3727+
return func() {
3728+
p = getGatewayPod(f, servingNamespace, gatewayPodName)
3729+
p.Finalizers = []string{}
3730+
updatePod(f, p)
3731+
}
3732+
case GatewayNotReady:
3733+
ginkgo.By("Remove /tmp/ready in external gateway pod so that readiness probe fails")
3734+
_, err = e2ekubectl.RunKubectl(servingNamespace, "exec", gatewayPodName, "--", "rm", "/tmp/ready")
3735+
framework.ExpectNoError(err, fmt.Sprintf("unable to remove /tmp/ready in pod: %s, err: %v", gatewayPodName, err))
3736+
gomega.Eventually(func() bool {
3737+
var p *corev1.Pod
3738+
p, err = f.ClientSet.CoreV1().Pods(servingNamespace).Get(context.Background(), gatewayPodName, metav1.GetOptions{})
3739+
if err != nil {
3740+
return false
3741+
}
3742+
podReadyStatus := corev1.ConditionTrue
3743+
for _, condition := range p.Status.Conditions {
3744+
if condition.Type == corev1.PodReady {
3745+
podReadyStatus = condition.Status
3746+
break
3747+
}
3748+
}
3749+
return podReadyStatus == corev1.ConditionFalse
3750+
}).WithTimeout(5*time.Minute).Should(gomega.Equal(true), fmt.Sprintf("Mark second external gateway pod %s from ns %s not ready, failed: %v", gatewayPodName, servingNamespace, err))
3751+
return nil
3752+
default:
3753+
framework.Failf("unexpected GatewayRemovalType passed: %s", removalType)
3754+
return nil
3755+
}
3756+
}

0 commit comments

Comments
 (0)