test: ensure LoadBalancer Service IP is reachable

Author: dkoshkin

test/e2e/config/caren.yaml

@@ -226,6 +226,7 @@ intervals:
   default/wait-deployment: ["10m", "10s"]
   default/wait-daemonset: [ "5m", "10s" ]
   default/wait-statefulset: [ "10m", "10s" ]
+  default/wait-service: [ "10m", "10s" ]
   default/wait-clusterresourceset: [ "5m", "10s" ]
   default/wait-helmrelease: [ "5m", "10s" ]
   default/wait-resource: [ "5m", "10s" ]

test/e2e/quick_start_test.go

@@ -96,17 +96,19 @@ var _ = Describe("Quick start", func() {
 												)
 											}
 
-											// For Nutanix provider, reserve an IP address for the workload cluster control plane endpoint -
-											// remember to unreserve it!
+											// For Nutanix provider, reserve an IP address for the workload cluster:
+											// 1. control plane endpoint
+											// 2. service load balancer
+											// Remember to unreserve it after the test!
 											if provider == "Nutanix" {
-												By(
-													"Reserving an IP address for the workload cluster control plane endpoint",
-												)
 												nutanixClient, err := nutanix.NewV4Client(
 													nutanix.CredentialsFromCAPIE2EConfig(testE2EConfig),
 												)
 												Expect(err).ToNot(HaveOccurred())
 
+												By(
+													"Reserving an IP address for the workload cluster control plane endpoint",
+												)
 												controlPlaneEndpointIP, unreserveControlPlaneEndpointIP, err := nutanix.ReserveIP(
 													testE2EConfig.MustGetVariable("NUTANIX_SUBNET_NAME"),
 													testE2EConfig.MustGetVariable(
@@ -117,6 +119,20 @@ var _ = Describe("Quick start", func() {
 												Expect(err).ToNot(HaveOccurred())
 												DeferCleanup(unreserveControlPlaneEndpointIP)
 												testE2EConfig.Variables["CONTROL_PLANE_ENDPOINT_IP"] = controlPlaneEndpointIP
+
+												By(
+													"Reserving an IP address for the workload cluster kubernetes Service load balancer",
+												)
+												kubernetesServiceLoadBalancerIP, unreservekubernetesServiceLoadBalancerIP, err := nutanix.ReserveIP(
+													testE2EConfig.MustGetVariable("NUTANIX_SUBNET_NAME"),
+													testE2EConfig.MustGetVariable(
+														"NUTANIX_PRISM_ELEMENT_CLUSTER_NAME",
+													),
+													nutanixClient,
+												)
+												Expect(err).ToNot(HaveOccurred())
+												DeferCleanup(unreservekubernetesServiceLoadBalancerIP)
+												testE2EConfig.Variables["KUBERNETES_SERVICE_LOAD_BALANCER_IP"] = kubernetesServiceLoadBalancerIP
 											}
 
 											clusterLocalTempDir, err := os.MkdirTemp("", "clusterctl-")
@@ -326,7 +342,6 @@ var _ = Describe("Quick start", func() {
 															ClusterProxy:    proxy,
 														},
 													)
-
 													EnsureAntiAffnityForRegistryAddon(
 														ctx,
 														EnsureAntiAffnityForRegistryAddonInput{
@@ -335,6 +350,21 @@ var _ = Describe("Quick start", func() {
 															ClusterProxy:    proxy,
 														},
 													)
+
+													// TODO: Test for other providers.
+													if provider == "Nutanix" {
+														EnsureLoadBalancerService(
+															ctx,
+															EnsureLoadBalancerServiceInput{
+																WorkloadCluster: workloadCluster,
+																ClusterProxy:    proxy,
+																ServciceIntervals: testE2EConfig.GetIntervals(
+																	flavor,
+																	"wait-service",
+																),
+															},
+														)
+													}
 												},
 											}
 										})

test/e2e/serviceloadbalancer_helpers.go

@@ -8,11 +8,19 @@ package e2e
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/utils/ptr"
 	clusterv1 "sigs.k8s.io/cluster-api/api/v1beta1"
 	"sigs.k8s.io/cluster-api/test/framework"
 	"sigs.k8s.io/controller-runtime/pkg/client"
@@ -136,3 +144,158 @@ func waitForMetalLBServiceLoadBalancerToBeReadyInWorkloadCluster(
 		Resources: resources,
 	}, input.resourceIntervals...)
 }
+
+type EnsureLoadBalancerServiceInput struct {
+	WorkloadCluster   *clusterv1.Cluster
+	ClusterProxy      framework.ClusterProxy
+	ServciceIntervals []interface{}
+}
+
+// EnsureLoadBalancerService creates a test Service of type LoadBalancer and tests that the assigned IP responds.
+func EnsureLoadBalancerService(
+	ctx context.Context,
+	input EnsureLoadBalancerServiceInput,
+) {
+	workloadClusterClient := input.ClusterProxy.GetWorkloadCluster(
+		ctx, input.WorkloadCluster.Namespace, input.WorkloadCluster.Name,
+	).GetClient()
+
+	svc := createTestService(ctx, workloadClusterClient, input.ServciceIntervals)
+	url := "http://" + getLoadBalancerAddress(svc) + "/clientip"
+
+	By("Testing the LoadBalancer Service responds")
+	url := url.URL{
+		Scheme: "http",
+		Host:   getLoadBalancerAddress(svc),
+		Path:   "/clientip",
+	}
+	output := testServiceLoadBalancer(ctx, url, input.ServciceIntervals)
+	Expect(output).ToNot(BeEmpty())
+}
+
+func createTestService(
+	ctx context.Context,
+	workloadClusterClient client.Client,
+	intervals []interface{},
+) *corev1.Service {
+	const (
+		name      = "echo"
+		namespace = corev1.NamespaceDefault
+		appKey    = "app"
+		replicas  = int32(1)
+		image     = "registry.k8s.io/e2e-test-images/agnhost:2.57"
+
+		port     = 8080
+		portName = "http"
+	)
+
+	By("Creating a test Deployment for LoadBalancer Service")
+	dep := &appsv1.Deployment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: appsv1.DeploymentSpec{
+			Replicas: ptr.To(replicas),
+			Selector: &metav1.LabelSelector{
+				MatchLabels: map[string]string{appKey: name},
+			},
+			Template: corev1.PodTemplateSpec{
+				ObjectMeta: metav1.ObjectMeta{
+					Labels: map[string]string{appKey: name},
+				},
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{{
+						Name:  name,
+						Image: image,
+						Args:  []string{"netexec", fmt.Sprintf("--http-port=%d", port)},
+						Ports: []corev1.ContainerPort{{
+							Name:          portName,
+							ContainerPort: int32(port),
+						}},
+					}},
+				},
+			},
+		},
+	}
+	if err := workloadClusterClient.Create(ctx, dep); err != nil {
+		Expect(err).ToNot(HaveOccurred())
+	}
+	By("Waiting for Deployment to be ready")
+	Eventually(func(g Gomega) {
+		g.Expect(workloadClusterClient.Get(ctx, client.ObjectKeyFromObject(dep), dep)).To(Succeed())
+		g.Expect(dep.Status.ReadyReplicas).To(Equal(replicas))
+	}, intervals...).Should(Succeed(), "timed out waiting for Deployment to be ready")
+
+	By("Creating a test Service for LoadBalancer Service")
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      name,
+			Namespace: namespace,
+		},
+		Spec: corev1.ServiceSpec{
+			Type:     corev1.ServiceTypeLoadBalancer,
+			Selector: map[string]string{appKey: name},
+			Ports: []corev1.ServicePort{{
+				Name:       portName,
+				Port:       80,
+				Protocol:   corev1.ProtocolTCP,
+				TargetPort: intstr.FromInt(port),
+			}},
+		},
+	}
+	if err := workloadClusterClient.Create(ctx, svc); err != nil {
+		Expect(err).ToNot(HaveOccurred())
+	}
+
+	key := client.ObjectKeyFromObject(svc)
+	By("Waiting for LoadBalacer IP/Hostname to be assigned")
+	Eventually(func(g Gomega) {
+		g.Expect(workloadClusterClient.Get(ctx, key, svc)).To(Succeed())
+
+		ings := svc.Status.LoadBalancer.Ingress
+		g.Expect(ings).ToNot(BeEmpty(), "no LoadBalancer ingress yet")
+
+		ip := ings[0].IP
+		host := ings[0].Hostname
+		g.Expect(ip == "" && host == "").To(BeFalse(), "ingress has neither IP nor Hostname yet")
+	}, intervals...).Should(Succeed(), "timed out waiting for LoadBalancer IP/hostname")
+
+	return svc
+}
+
+func getLoadBalancerAddress(svc *corev1.Service) string {
+	ings := svc.Status.LoadBalancer.Ingress
+	if len(ings) == 0 {
+		return ""
+	}
+	address := ings[0].IP
+	if address == "" {
+		address = ings[0].Hostname
+	}
+	return address
+}
+
+func testServiceLoadBalancer(
+	ctx context.Context,
+	url url.URL,
+	intervals []interface{},
+) string {
+	hc := &http.Client{Timeout: 5 * time.Second}
+	var output string
+	Eventually(func(g Gomega) string {
+		req, _ := http.NewRequestWithContext(ctx, http.MethodGet, url.String(), http.NoBody)
+		resp, err := hc.Do(req)
+		if err != nil {
+			return ""
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != http.StatusOK {
+			return ""
+		}
+		b, _ := io.ReadAll(resp.Body)
+		output = strings.TrimSpace(string(b))
+		return output
+	}, intervals...).ShouldNot(BeEmpty(), "no response from service")
+	return output
+}