e2e: Move underlay setup to providers

Signed-off-by: Enrique Llorente <[email protected]>

Author: qinqon

test/e2e/infraprovider/api/api.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+
+	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 // Provider represents the infrastructure provider
@@ -37,6 +39,21 @@ type Provider interface {
 	GetK8HostPort() uint16 // supported K8 host ports
 }
 
+// Underlay represents the configuration for an underlay network.
+// Note: The physical network referenced by PhysicalNetworkName must be pre-created and available.
+type Underlay struct {
+	// PhysicalNetworkName is the name of the pre-created physical network to use.
+	PhysicalNetworkName string
+	// LogicalNetworkName is the logical network name to be used.
+	LogicalNetworkName string
+	// BridgeName is the name of the bridge associated with the underlay.
+	BridgeName string
+	// PortName is the name of the port on the bridge.
+	PortName string
+	// VlanID is the VLAN identifier for the underlay network.
+	VlanID int
+}
+
 type Context interface {
 	CreateExternalContainer(container ExternalContainer) (ExternalContainer, error)
 	DeleteExternalContainer(container ExternalContainer) error
@@ -46,6 +63,7 @@ type Context interface {
 	AttachNetwork(network Network, instance string) (NetworkInterface, error)
 	DetachNetwork(network Network, instance string) error
 	GetAttachedNetworks() (Networks, error)
+	SetupUnderlay(f *framework.Framework, underlay Underlay) error
 
 	AddCleanUpFn(func() error)
 }

test/e2e/infraprovider/providers/kind/kind.go

@@ -13,10 +13,12 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/containerengine"
+	"github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/images"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/api"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/portalloc"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 	utilnet "k8s.io/utils/net"
@@ -359,6 +361,83 @@ func (c *contextKind) getAttachedNetworks() (api.Networks, error) {
 	return attachedNetworks, nil
 }
 
+func (c *contextKind) SetupUnderlay(f *framework.Framework, underlay api.Underlay) error {
+	if underlay.LogicalNetworkName == "" {
+		return fmt.Errorf("underlay logical network name must be set")
+	}
+
+	if underlay.PhysicalNetworkName == "" {
+		underlay.PhysicalNetworkName = "underlay"
+	}
+
+	if underlay.BridgeName == "" {
+		underlay.BridgeName = secondaryBridge
+	}
+
+	const (
+		ovsKubeNodeLabel = "app=ovnkube-node"
+	)
+
+	ovsPodList, err := f.ClientSet.CoreV1().Pods(deploymentconfig.Get().OVNKubernetesNamespace()).List(
+		context.Background(),
+		metav1.ListOptions{LabelSelector: ovsKubeNodeLabel},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to list OVS pods with label %q at namespace %q: %w", ovsKubeNodeLabel, deploymentconfig.Get().OVNKubernetesNamespace(), err)
+	}
+
+	if len(ovsPodList.Items) == 0 {
+		return fmt.Errorf("no pods with label %q in namespace %q", ovsKubeNodeLabel, deploymentconfig.Get().OVNKubernetesNamespace())
+	}
+	for _, ovsPod := range ovsPodList.Items {
+		if underlay.BridgeName != deploymentconfig.Get().ExternalBridgeName() {
+			underlayInterface, err := getNetworkInterface(ovsPod.Spec.NodeName, underlay.PhysicalNetworkName)
+			if err != nil {
+				return fmt.Errorf("failed to get underlay interface for network %s on node %s: %w", underlay.PhysicalNetworkName, ovsPod.Spec.NodeName, err)
+			}
+			c.AddCleanUpFn(func() error {
+				if err := removeOVSBridge(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName); err != nil {
+					return fmt.Errorf("failed to remove OVS bridge %s for pod %s/%s during cleanup: %w", underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+				}
+				return nil
+			})
+			if err := ensureOVSBridge(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName); err != nil {
+				return fmt.Errorf("failed to add OVS bridge %s for pod %s/%s: %w", underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+			}
+
+			if err := ovsAttachPortToBridge(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName, underlayInterface.InfName); err != nil {
+				return fmt.Errorf("failed to attach port %s to bridge %s for pod %s/%s: %w", underlayInterface.InfName, underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+			}
+			if underlay.VlanID > 0 {
+				if err := ovsEnableVLANAccessPort(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName, underlayInterface.InfName, underlay.VlanID); err != nil {
+					return fmt.Errorf("failed to enable VLAN %d on port %s for bridge %s for pod %s/%s: %w", underlay.VlanID, underlayInterface.InfName, underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+				}
+			}
+		}
+		c.AddCleanUpFn(func() error {
+			if err := configureBridgeMappings(
+				ovsPod.Namespace,
+				ovsPod.Name,
+				defaultNetworkBridgeMapping(),
+			); err != nil {
+				return fmt.Errorf("failed to restore default bridge mappings for pod %s/%s during cleanup: %w", ovsPod.Namespace, ovsPod.Name, err)
+			}
+			return nil
+		})
+
+		if err := configureBridgeMappings(
+			ovsPod.Namespace,
+			ovsPod.Name,
+			defaultNetworkBridgeMapping(),
+			bridgeMapping(underlay.LogicalNetworkName, underlay.BridgeName),
+		); err != nil {
+			return fmt.Errorf("failed to configure bridge mappings for pod %s/%s for logical network %s to bridge %s: %w", ovsPod.Namespace, ovsPod.Name, underlay.LogicalNetworkName, underlay.BridgeName, err)
+		}
+	}
+	return nil
+
+}
+
 func (c *contextKind) AddCleanUpFn(cleanUpFn func() error) {
 	c.Lock()
 	defer c.Unlock()

test/e2e/infraprovider/providers/kind/ovs.go

@@ -0,0 +1,93 @@
+package kind
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig"
+
+	e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
+)
+
+const (
+	secondaryBridge = "ovsbr1"
+)
+
+func ensureOVSBridge(podNamespace, podName string, bridgeName string) error {
+	cmd := fmt.Sprintf("ovs-vsctl br-exists %[1]s || ovs-vsctl add-br %[1]s", bridgeName)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to add ovs bridge %q: %v", bridgeName, err)
+	}
+	return nil
+}
+
+func removeOVSBridge(podNamespace, podName string, bridgeName string) error {
+	cmd := fmt.Sprintf("if ovs-vsctl br-exists %[1]s; then ovs-vsctl del-br %[1]s; fi", bridgeName)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to remove ovs bridge %q: %v", bridgeName, err)
+	}
+	return nil
+}
+
+func ovsAttachPortToBridge(podNamespace, podName string, bridgeName string, portName string) error {
+	cmd := fmt.Sprintf("ovs-vsctl list port %[2]s || ovs-vsctl add-port %[1]s %[2]s", bridgeName, portName)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to addadd  port %s from OVS bridge %s: %v", portName, bridgeName, err)
+	}
+	return nil
+}
+
+func ovsEnableVLANAccessPort(podNamespace, podName string, bridgeName string, portName string, vlanID int) error {
+	cmd := fmt.Sprintf("ovs-vsctl set port %[1]s tag=%[2]d vlan_mode=access", portName, vlanID)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to enable vlan access port %s from OVS bridge %s: %v", portName, bridgeName, err)
+	}
+	return nil
+}
+
+type BridgeMapping struct {
+	physnet   string
+	ovsBridge string
+}
+
+func (bm BridgeMapping) String() string {
+	return fmt.Sprintf("%s:%s", bm.physnet, bm.ovsBridge)
+}
+
+type BridgeMappings []BridgeMapping
+
+func (bms BridgeMappings) String() string {
+	return strings.Join(Map(bms, func(bm BridgeMapping) string { return bm.String() }), ",")
+}
+
+func Map[T, V any](items []T, fn func(T) V) []V {
+	result := make([]V, len(items))
+	for i, t := range items {
+		result[i] = fn(t)
+	}
+	return result
+}
+
+func configureBridgeMappings(podNamespace, podName string, mappings ...BridgeMapping) error {
+	mappingsString := fmt.Sprintf("external_ids:ovn-bridge-mappings=%s", BridgeMappings(mappings).String())
+	cmd := strings.Join([]string{"ovs-vsctl", "set", "open", ".", mappingsString}, " ")
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to configure bridge mappings %q: %v", mappingsString, err)
+	}
+	return nil
+}
+
+func defaultNetworkBridgeMapping() BridgeMapping {
+	return BridgeMapping{
+		physnet:   "physnet",
+		ovsBridge: deploymentconfig.Get().ExternalBridgeName(),
+	}
+}
+
+func bridgeMapping(physnet, ovsBridge string) BridgeMapping {
+	return BridgeMapping{
+		physnet:   physnet,
+		ovsBridge: ovsBridge,
+	}
+}

test/e2e/kubevirt.go

@@ -1721,17 +1721,7 @@ write_files:
 
 			if td.topology == udnv1.NetworkTopologyLocalnet {
 				By("setting up the localnet underlay")
-				nodes := ovsPods(clientSet)
-				Expect(nodes).NotTo(BeEmpty())
-				DeferCleanup(func() {
-					if e2eframework.TestContext.DeleteNamespace && (e2eframework.TestContext.DeleteNamespaceOnFailure || !CurrentSpecReport().Failed()) {
-						By("tearing down the localnet underlay")
-						Expect(teardownUnderlay(nodes, secondaryBridge)).To(Succeed())
-					}
-				})
-
-				const secondaryInterfaceName = "eth1"
-				Expect(setupUnderlay(nodes, secondaryBridge, secondaryInterfaceName, networkName, 0 /*vlanID*/)).To(Succeed())
+				Expect(providerCtx.SetupUnderlay(fr, infraapi.Underlay{LogicalNetworkName: networkName})).To(Succeed())
 			}
 			createCUDN(cudn)
 
@@ -2209,20 +2199,10 @@ chpasswd: { expire: False }
 		)
 		DescribeTable("should maintain tcp connection with minimal downtime", func(td func(vmi *kubevirtv1.VirtualMachineInstance)) {
 			By("setting up the localnet underlay")
-			nodes := ovsPods(clientSet)
-			Expect(nodes).NotTo(BeEmpty())
-			DeferCleanup(func() {
-				if e2eframework.TestContext.DeleteNamespace && (e2eframework.TestContext.DeleteNamespaceOnFailure || !CurrentSpecReport().Failed()) {
-					By("tearing down the localnet underlay")
-					Expect(teardownUnderlay(nodes, secondaryBridge)).To(Succeed())
-				}
-			})
-
 			cudn, networkName := kubevirt.GenerateCUDN(namespace, "net1", udnv1.NetworkTopologyLocalnet, udnv1.NetworkRoleSecondary, udnv1.DualStackCIDRs{})
 			createCUDN(cudn)
 
-			const secondaryInterfaceName = "eth1"
-			Expect(setupUnderlay(nodes, secondaryBridge, secondaryInterfaceName, networkName, 0 /*vlanID*/)).To(Succeed())
+			Expect(providerCtx.SetupUnderlay(fr, infraapi.Underlay{LogicalNetworkName: networkName})).To(Succeed())
 
 			workerNodeList, err := fr.ClientSet.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{LabelSelector: labels.FormatLabels(map[string]string{"node-role.kubernetes.io/worker": ""})})
 			Expect(err).NotTo(HaveOccurred())