e2e: Move http servers to external container

Signed-off-by: Enrique Llorente <[email protected]>

Author: qinqon

test/e2e/infraprovider/api/api.go

@@ -182,13 +182,14 @@ func (n NetworkInterface) GetMAC() string {
 }
 
 type ExternalContainer struct {
-	Name    string
-	Image   string
-	Network Network
-	Args    []string
-	ExtPort uint16
-	IPv4    string
-	IPv6    string
+	Name       string
+	Image      string
+	Network    Network
+	Entrypoint string
+	Args       []string
+	ExtPort    uint16
+	IPv4       string
+	IPv6       string
 }
 
 func (ec ExternalContainer) GetName() string {

test/e2e/infraprovider/providers/kind/kind.go

@@ -147,6 +147,9 @@ func (c *contextKind) createExternalContainer(container api.ExternalContainer) (
 		return container, fmt.Errorf("container %s already exists", container.Name)
 	}
 	cmd := []string{"run", "-itd", "--privileged", "--name", container.Name, "--network", container.Network.Name(), "--hostname", container.Name}
+	if container.Entrypoint != "" {
+		cmd = append(cmd, "--entrypoint", container.Entrypoint)
+	}
 	cmd = append(cmd, container.Image)
 	if len(container.Args) > 0 {
 		cmd = append(cmd, container.Args...)

test/e2e/localnet-underlay.go

@@ -1,52 +1 @@
 package e2e
-
-import (
-	"fmt"
-	"os"
-	"os/exec"
-)
-
-// TODO: make this function idempotent; use golang netlink instead
-func createVLANInterface(deviceName string, vlanID string, ipAddress *string) error {
-	vlan := vlanName(deviceName, vlanID)
-	cmd := exec.Command("sudo", "ip", "link", "add", "link", deviceName, "name", vlan, "type", "vlan", "id", vlanID)
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed to create vlan interface %s: %v", vlan, err)
-	}
-
-	cmd = exec.Command("sudo", "ip", "link", "set", "dev", vlan, "up")
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed to enable vlan interface %s: %v", vlan, err)
-	}
-
-	if ipAddress != nil {
-		cmd = exec.Command("sudo", "ip", "addr", "add", *ipAddress, "dev", vlan)
-		cmd.Stderr = os.Stderr
-
-		if err := cmd.Run(); err != nil {
-			return fmt.Errorf("failed to define the vlan interface %q IP Address %s: %v", vlan, *ipAddress, err)
-		}
-	}
-	return nil
-}
-
-// TODO: make this function idempotent; use golang netlink instead
-func deleteVLANInterface(deviceName string, vlanID string) error {
-	vlan := vlanName(deviceName, vlanID)
-	cmd := exec.Command("sudo", "ip", "link", "del", vlan)
-	cmd.Stderr = os.Stderr
-	if err := cmd.Run(); err != nil {
-		return fmt.Errorf("failed to delete vlan interface %s: %v", vlan, err)
-	}
-	return nil
-}
-
-func vlanName(deviceName string, vlanID string) string {
-	// MAX IFSIZE 16; got to truncate it to add the vlan suffix
-	if len(deviceName)+len(vlanID)+1 > 16 {
-		deviceName = deviceName[:len(deviceName)-len(vlanID)-1]
-	}
-	return fmt.Sprintf("%s.%s", deviceName, vlanID)
-}

test/e2e/multihoming.go

@@ -5,9 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"net/netip"
-	"os"
-	"os/exec"
-	"strconv"
 	"strings"
 	"time"
 
@@ -16,7 +13,6 @@ import (
 	. "github.com/onsi/gomega"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/feature"
 
-	"github.com/docker/docker/client"
 	v1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -32,6 +28,7 @@ import (
 	ipgenerator "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/generator/ip"
 	util "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig"
+	"github.com/ovn-org/ovn-kubernetes/test/e2e/images"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider"
 	infraapi "github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/api"
 )
@@ -907,9 +904,9 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() {
 				expectedOriginalMTU        = 1200
 			)
 
-			var netConfig networkAttachmentConfig
-			var underlayBridgeName string
-			var cmdWebServer *exec.Cmd
+			var (
+				netConfig networkAttachmentConfig
+			)
 
 			underlayIP := underlayServiceIP + "/24"
 			Context("with a service running on the underlay", func() {
@@ -932,28 +929,23 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() {
 					})).To(Succeed())
 				})
 
-				BeforeEach(func() {
-					By("adding IP to the underlay docker bridge")
-					cli, err := client.NewClientWithOpts(client.FromEnv)
-					Expect(err).NotTo(HaveOccurred())
-
-					gatewayIP, err := getNetworkGateway(cli, dockerNetworkName)
-					Expect(err).NotTo(HaveOccurred())
-
-					underlayBridgeName, err = findInterfaceByIP(gatewayIP)
-					Expect(err).NotTo(HaveOccurred())
-
-					cmd := exec.Command("sudo", "ip", "addr", "add", underlayIP, "dev", underlayBridgeName)
-					cmd.Stderr = os.Stderr
-					err = cmd.Run()
-					Expect(err).NotTo(HaveOccurred())
-				})
-
 				BeforeEach(func() {
 					By("starting a service, connected to the underlay")
-					cmdWebServer = exec.Command("python3", "-m", "http.server", "--bind", underlayServiceIP, strconv.Itoa(int(servicePort)))
-					cmdWebServer.Stderr = os.Stderr
-					Expect(cmdWebServer.Start()).NotTo(HaveOccurred(), "failed to create web server, port might be busy")
+					providerCtx = infraprovider.Get().NewTestContext()
+
+					underlayNetwork, err := infraprovider.Get().GetNetwork(dockerNetworkName)
+					Expect(err).NotTo(HaveOccurred(), "must get underlay network")
+					externalContainerName := f.Namespace.Name + "-web-server"
+					serviceContainerSpec := infraapi.ExternalContainer{
+						Name:       externalContainerName,
+						Image:      images.AgnHost(),
+						Network:    underlayNetwork,
+						Entrypoint: "bash",
+						Args:       []string{"-c", fmt.Sprintf("ip a add %s/24 dev eth0 && ./agnhost netexec --http-port=%d", underlayServiceIP, servicePort)},
+						ExtPort:    servicePort,
+					}
+					_, err = providerCtx.CreateExternalContainer(serviceContainerSpec)
+					Expect(err).NotTo(HaveOccurred(), "must create external container 1")
 				})
 
 				BeforeEach(func() {
@@ -966,18 +958,6 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() {
 					Expect(err).NotTo(HaveOccurred())
 				})
 
-				AfterEach(func() {
-					err := cmdWebServer.Process.Kill()
-					Expect(err).NotTo(HaveOccurred())
-				})
-
-				AfterEach(func() {
-					cmd := exec.Command("sudo", "ip", "addr", "del", underlayIP, "dev", underlayBridgeName)
-					cmd.Stderr = os.Stderr
-					err := cmd.Run()
-					Expect(err).NotTo(HaveOccurred())
-				})
-
 				It("correctly sets the MTU on the pod", func() {
 					Eventually(func() error {
 						clientPodConfig := podConfiguration{
@@ -1008,6 +988,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() {
 
 					By("asserting the *client* pod can contact the underlay service")
 					Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(Succeed())
+
 				})
 
 				Context("and networkAttachmentDefinition is modified", func() {
@@ -1315,29 +1296,30 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() {
 						VlanID:             netConfig.vlanID,
 					})).To(Succeed(), "configuring the OVS bridge")
 
-					By(fmt.Sprintf("creating a VLAN interface on top of the bridge connecting the cluster nodes with IP: %s", underlayIP))
-					cli, err := client.NewClientWithOpts(client.FromEnv)
-					Expect(err).NotTo(HaveOccurred())
-
-					gatewayIP, err := getNetworkGateway(cli, dockerNetworkName)
-					Expect(err).NotTo(HaveOccurred())
-
-					underlayBridgeName, err = findInterfaceByIP(gatewayIP)
-					Expect(err).NotTo(HaveOccurred())
-					Expect(createVLANInterface(underlayBridgeName, strconv.Itoa(vlanID), &underlayIP)).To(
-						Succeed(),
-						"create a VLAN interface on the bridge interconnecting the cluster nodes",
-					)
-
-					By("starting a service, connected to the underlay")
-					cmdWebServer = exec.Command("python3", "-m", "http.server", "--bind", underlayServiceIP, strconv.Itoa(port))
-					cmdWebServer.Stderr = os.Stderr
-					Expect(cmdWebServer.Start()).NotTo(HaveOccurred(), "failed to create web server, port might be busy")
-				})
+					By("starting a service, connected to the underlay over a VLAN")
+					providerCtx = infraprovider.Get().NewTestContext()
+
+					ifName := "eth0"
+					vlanName := fmt.Sprintf("%s.%d", ifName, vlanID)
+					underlayNetwork, err := infraprovider.Get().GetNetwork(dockerNetworkName)
+					Expect(err).NotTo(HaveOccurred(), "must get underlay network")
+					externalContainerName := f.Namespace.Name + "-web-server"
+					serviceContainerSpec := infraapi.ExternalContainer{
+						Name:       externalContainerName,
+						Image:      images.AgnHost(),
+						Network:    underlayNetwork,
+						Entrypoint: "bash",
+						ExtPort:    servicePort,
+						Args: []string{"-c", fmt.Sprintf(`
+ip link add link %[1]s name %[2]s type vlan id %[3]d
+ip link set dev %[2]s up
+ip a add %[4]s/24 dev %[2]s
+./agnhost netexec --http-port=%[5]d
+`, ifName, vlanName, vlanID, underlayServiceIP, servicePort)},
+					}
+					_, err = providerCtx.CreateExternalContainer(serviceContainerSpec)
+					Expect(err).NotTo(HaveOccurred(), "must create external container 1")
 
-				AfterEach(func() {
-					Expect(cmdWebServer.Process.Kill()).NotTo(HaveOccurred(), "kill the python webserver")
-					Expect(deleteVLANInterface(underlayBridgeName, strconv.Itoa(vlanID))).NotTo(HaveOccurred(), "remove the underlay physical configuration")
 				})
 
 				It("the same bridge mapping can be shared by a separate VLAN by using the physical network name attribute", func() {
@@ -1370,6 +1352,7 @@ var _ = Describe("Multi Homing", feature.MultiHoming, func() {
 
 					By(fmt.Sprintf("asserting the *client* pod can contact the underlay service with IP %q on the separate vlan", underlayIP))
 					Expect(connectToServer(clientPodConfig, underlayServiceIP, servicePort)).To(Succeed())
+
 				})
 			})
 		})