Merge pull request kubernetes#131538 from aojea/blue_green_e2e

e2e network: blue green deployments

Author: k8s-ci-robot

test/e2e/network/pod_lifecycle.go

@@ -21,13 +21,20 @@ import (
 	"fmt"
 	"net"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/onsi/ginkgo/v2"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	types "k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/kubernetes/test/e2e/framework"
 	e2epod "k8s.io/kubernetes/test/e2e/framework/pod"
+	e2eoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
+	e2eservice "k8s.io/kubernetes/test/e2e/framework/service"
 	"k8s.io/kubernetes/test/e2e/network/common"
 	admissionapi "k8s.io/pod-security-admission/api"
 )
@@ -98,4 +105,234 @@ var _ = common.SIGDescribe("Connectivity Pod Lifecycle", func() {
 		execHostnameTest(*pausePod, podIPAddress, webserverPod.Name)
 	})
 
+	ginkgo.It("should be able to have zero downtime on a Blue Green deployment using Services and Readiness Gates", func(ctx context.Context) {
+		readinessGate := "k8s.io/blue-green"
+		patchStatusFmt := `{"status":{"conditions":[{"type":%q, "status":%q}]}}`
+
+		serviceName := "blue-green-svc"
+		blueGreenJig := e2eservice.NewTestJig(cs, ns, serviceName)
+		ginkgo.By("creating a service " + serviceName + " with type=ClusterIP in " + ns)
+		blueGreenService, err := blueGreenJig.CreateTCPService(ctx, func(svc *v1.Service) {
+			svc.Spec.Type = v1.ServiceTypeClusterIP
+			svc.Spec.Ports = []v1.ServicePort{
+				{Port: 80, Name: "http", Protocol: v1.ProtocolTCP, TargetPort: intstr.FromInt32(80)},
+			}
+		})
+		framework.ExpectNoError(err)
+
+		ginkgo.By("Creating 2 webserver pod (green and blue) able to serve traffic during the grace period of 30 seconds")
+		gracePeriod := int64(30)
+		bluePod := e2epod.NewAgnhostPod(ns, "blue-pod", nil, nil, nil, "netexec", "--http-port=80", fmt.Sprintf("--delay-shutdown=%d", gracePeriod))
+		bluePod.Labels = blueGreenService.Labels
+		bluePod.Spec.ReadinessGates = []v1.PodReadinessGate{
+			{ConditionType: v1.PodConditionType(readinessGate)},
+		}
+		podClient.Create(ctx, bluePod)
+		err = e2epod.WaitForPodNameRunningInNamespace(ctx, cs, bluePod.Name, ns)
+		if err != nil {
+			framework.Failf("waiting for pod %s : %v", bluePod.Name, err)
+		}
+		if podClient.PodIsReady(ctx, bluePod.Name) {
+			framework.Failf("Expect pod(%s/%s)'s Ready condition to be false initially.", ns, bluePod.Name)
+		}
+
+		greenPod := e2epod.NewAgnhostPod(ns, "green-pod", nil, nil, nil, "netexec", "--http-port=80", fmt.Sprintf("--delay-shutdown=%d", gracePeriod))
+		greenPod.Labels = blueGreenService.Labels
+		greenPod.Spec.ReadinessGates = []v1.PodReadinessGate{
+			{ConditionType: v1.PodConditionType(readinessGate)},
+		}
+		podClient.Create(ctx, greenPod)
+		err = e2epod.WaitForPodNameRunningInNamespace(ctx, cs, greenPod.Name, ns)
+		if err != nil {
+			framework.Failf("waiting for pod %s : %v", greenPod.Name, err)
+		}
+		if podClient.PodIsReady(ctx, greenPod.Name) {
+			framework.Failf("Expect pod(%s/%s)'s Ready condition to be false initially.", ns, greenPod.Name)
+		}
+
+		ginkgo.By("Creating 1 client pod that will try to connect to the blue-green-svc")
+		clientPod := e2epod.NewAgnhostPod(ns, "client-pod-1", nil, nil, nil)
+		clientPod.Spec.TerminationGracePeriodSeconds = &gracePeriod
+		clientPod = podClient.CreateSync(ctx, clientPod)
+
+		ginkgo.By(fmt.Sprintf("patching blue pod status with condition %q to true", readinessGate))
+		_, err = podClient.Patch(ctx, bluePod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate, "True")), metav1.PatchOptions{}, "status")
+		if err != nil {
+			framework.Failf("failed to patch %s pod condition: %v", bluePod.Name, err)
+		}
+
+		// Expect EndpointSlice resource to have the blue pod ready to serve traffic
+		if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, wait.ForeverTestTimeout, true, func(context.Context) (bool, error) {
+			endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(blueGreenJig.Namespace).List(ctx, metav1.ListOptions{
+				LabelSelector: "kubernetes.io/service-name=" + blueGreenJig.Name,
+			})
+			if err != nil {
+				return false, err
+			}
+			for _, slice := range endpointSliceList.Items {
+				for _, ep := range slice.Endpoints {
+					if ep.TargetRef != nil &&
+						ep.TargetRef.Name == bluePod.Name &&
+						ep.TargetRef.Namespace == bluePod.Namespace &&
+						ep.Conditions.Ready != nil && *ep.Conditions.Ready {
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		}); err != nil {
+			framework.Failf("No EndpointSlice found for Service %s/%s: %s", blueGreenJig.Namespace, blueGreenJig.Name, err)
+		}
+
+		ginkgo.By("Try to connect to the blue pod through the service")
+		scvAddress := net.JoinHostPort(blueGreenService.Spec.ClusterIP, strconv.Itoa(80))
+		// assert 5 times that we can connect only to the blue pod
+		for i := 0; i < 5; i++ {
+			err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) {
+				cmd := fmt.Sprintf(`curl -q -s --connect-timeout 5 %s/hostname`, scvAddress)
+				stdout, err := e2eoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, cmd)
+				if err != nil {
+					framework.Logf("expected error when trying to connect to cluster IP : %v", err)
+					return false, nil
+				}
+				if strings.TrimSpace(stdout) == "" {
+					framework.Logf("got empty stdout, retry until timeout")
+					return false, nil
+				}
+				// Ensure we're comparing hostnames and not FQDNs
+				targetHostname := strings.Split(bluePod.Name, ".")[0]
+				hostname := strings.TrimSpace(strings.Split(stdout, ".")[0])
+				if hostname != targetHostname {
+					return false, fmt.Errorf("expecting hostname %s got %s", targetHostname, hostname)
+				}
+				return true, nil
+			})
+			if err != nil {
+				framework.Failf("can not connect to pod %s on address %s : %v", bluePod.Name, scvAddress, err)
+			}
+		}
+
+		// Switch from blue to green
+		ginkgo.By(fmt.Sprintf("patching green pod status with condition %q to true", readinessGate))
+		_, err = podClient.Patch(ctx, greenPod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate, "True")), metav1.PatchOptions{}, "status")
+		if err != nil {
+			framework.Failf("failed to patch %s pod condition: %v", greenPod.Name, err)
+		}
+
+		// Expect EndpointSlice resource to have the green pod ready to serve traffic
+		if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, wait.ForeverTestTimeout, true, func(context.Context) (bool, error) {
+			endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(blueGreenJig.Namespace).List(ctx, metav1.ListOptions{
+				LabelSelector: "kubernetes.io/service-name=" + blueGreenJig.Name,
+			})
+			if err != nil {
+				return false, err
+			}
+			for _, slice := range endpointSliceList.Items {
+				for _, ep := range slice.Endpoints {
+					if ep.TargetRef != nil &&
+						ep.TargetRef.Name == greenPod.Name &&
+						ep.TargetRef.Namespace == greenPod.Namespace &&
+						ep.Conditions.Ready != nil && *ep.Conditions.Ready {
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		}); err != nil {
+			framework.Failf("No EndpointSlice found for Service %s/%s: %s", blueGreenJig.Namespace, blueGreenJig.Name, err)
+		}
+
+		ginkgo.By(fmt.Sprintf("patching blue pod status with condition %q to false", readinessGate))
+		_, err = podClient.Patch(ctx, bluePod.Name, types.StrategicMergePatchType, []byte(fmt.Sprintf(patchStatusFmt, readinessGate, "False")), metav1.PatchOptions{}, "status")
+		if err != nil {
+			framework.Failf("failed to patch %s pod condition: %v", bluePod.Name, err)
+		}
+
+		// Expect EndpointSlice resource to have the blue pod NOT ready to serve traffic
+		if err := wait.PollUntilContextTimeout(ctx, 2*time.Second, wait.ForeverTestTimeout, true, func(context.Context) (bool, error) {
+			endpointSliceList, err := cs.DiscoveryV1().EndpointSlices(blueGreenJig.Namespace).List(ctx, metav1.ListOptions{
+				LabelSelector: "kubernetes.io/service-name=" + blueGreenJig.Name,
+			})
+			if err != nil {
+				return false, err
+			}
+			for _, slice := range endpointSliceList.Items {
+				for _, ep := range slice.Endpoints {
+					if ep.TargetRef != nil &&
+						ep.TargetRef.Name == bluePod.Name &&
+						ep.TargetRef.Namespace == bluePod.Namespace &&
+						ep.Conditions.Ready != nil && !*ep.Conditions.Ready {
+						return true, nil
+					}
+				}
+			}
+			return false, nil
+		}); err != nil {
+			framework.Failf("No EndpointSlice found for Service %s/%s: %s", blueGreenJig.Namespace, blueGreenJig.Name, err)
+		}
+
+		// We have checked the endpoint slices reflect the desired state:
+		// bluePod not ready and greenPod ready, but we need to remember kubernetes
+		// is a distributed system eventually consistent, so there is a propagation
+		// delay until this information is present on the nodes and a programming delay
+		// until the corresponding node components program the information on the dataplane.
+		err = wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) {
+			cmd := fmt.Sprintf(`curl -q -s --connect-timeout 5 %s/hostname`, scvAddress)
+			stdout, err := e2eoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, cmd)
+			if err != nil {
+				framework.Logf("expected error when trying to connect to cluster IP : %v", err)
+				return false, nil
+			}
+			if strings.TrimSpace(stdout) == "" {
+				framework.Logf("got empty stdout, retry until timeout")
+				return false, nil
+			}
+			// Ensure we're comparing hostnames and not FQDNs
+			targetHostname := strings.Split(greenPod.Name, ".")[0]
+			hostname := strings.TrimSpace(strings.Split(stdout, ".")[0])
+			if hostname != targetHostname {
+				framework.Logf("expecting hostname %s got %s", targetHostname, hostname)
+				return false, nil
+			}
+			return true, nil
+		})
+		if err != nil {
+			framework.Failf("can not connect to pod %s on address %s : %v", greenPod.Name, scvAddress, err)
+		}
+
+		ginkgo.By("Try to connect to the green pod through the service")
+		// assert 5 times that we can connect only to the green pod
+		for i := 0; i < 5; i++ {
+			err := wait.PollUntilContextTimeout(ctx, 3*time.Second, 30*time.Second, true, func(ctx context.Context) (done bool, err error) {
+				cmd := fmt.Sprintf(`curl -q -s --connect-timeout 5 %s/hostname`, scvAddress)
+				stdout, err := e2eoutput.RunHostCmd(clientPod.Namespace, clientPod.Name, cmd)
+				if err != nil {
+					framework.Logf("expected error when trying to connect to cluster IP : %v", err)
+					return false, nil
+				}
+				if strings.TrimSpace(stdout) == "" {
+					framework.Logf("got empty stdout, retry until timeout")
+					return false, nil
+				}
+				// Ensure we're comparing hostnames and not FQDNs
+				targetHostname := strings.Split(greenPod.Name, ".")[0]
+				hostname := strings.TrimSpace(strings.Split(stdout, ".")[0])
+				// At this point we should only receive traffic from the green Pod.
+				if hostname != targetHostname {
+					return false, fmt.Errorf("expecting hostname %s got %s", targetHostname, hostname)
+				}
+				return true, nil
+			})
+			if err != nil {
+				framework.Failf("can not connect to pod %s on address %s : %v", greenPod.Name, scvAddress, err)
+			}
+		}
+
+		// TODO there can be multiple combinations like:
+		// test zero downtime deleting the blue pod instead setting the readiness to false
+		// test roll back setting back the readiness to true on the blue pod
+		// ...
+
+	})
+
 })