Merge pull request #5220 from qinqon/kv-e2e-integrate-providers

Adapt e2e kind provider for kubevirt localnet tests

Author: trozet

contrib/kind-common

@@ -388,30 +388,7 @@ install_kubevirt() {
 
     local kubevirt_stable_release_url=$(get_kubevirt_release_url "stable")
     kubectl -n kubevirt patch kubevirt kubevirt --type=json --patch '[{"op":"add","path":"/spec/configuration/network","value":{}},{"op":"add","path":"/spec/configuration/network/binding","value":{"l2bridge":{"domainAttachmentType":"managedTap","migration":{}}}}]'
-    
-    if [ ! -d "./bin" ]
-    then
-        mkdir -p ./bin
-        if_error_exit "Failed to create bin dir!"
-    fi
-
-    if [[ "$OSTYPE" == "linux-gnu" ]]; then
-        OS_TYPE="linux"
-    elif [[ "$OSTYPE" == "darwin"* ]]; then
-        OS_TYPE="darwin"
-    fi
-
-    pushd ./bin
-       if [ ! -f ./virtctl ]; then
-           kubevirt_stable_release_url=$(get_kubevirt_release_url "stable")
-           cli_name="virtctl-${kubevirt_stable_release_url##*/}-${OS_TYPE}-${ARCH}"
-           curl -LO "${kubevirt_stable_release_url}/${cli_name}"
-           mv ${cli_name} virtctl
-           if_error_exit "Failed to download virtctl!"
-       fi
-    popd
 
-    chmod +x ./bin/virtctl
 }
 
 install_cert_manager() {

test/e2e/infraprovider/api/api.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+
+	"k8s.io/kubernetes/test/e2e/framework"
 )
 
 // Provider represents the infrastructure provider
@@ -37,6 +39,21 @@ type Provider interface {
 	GetK8HostPort() uint16 // supported K8 host ports
 }
 
+// Underlay represents the configuration for an underlay network.
+// Note: The physical network referenced by PhysicalNetworkName must be pre-created and available.
+type Underlay struct {
+	// PhysicalNetworkName is the name of the pre-created physical network to use.
+	PhysicalNetworkName string
+	// LogicalNetworkName is the logical network name to be used.
+	LogicalNetworkName string
+	// BridgeName is the name of the bridge associated with the underlay.
+	BridgeName string
+	// PortName is the name of the port on the bridge.
+	PortName string
+	// VlanID is the VLAN identifier for the underlay network.
+	VlanID int
+}
+
 type Context interface {
 	CreateExternalContainer(container ExternalContainer) (ExternalContainer, error)
 	DeleteExternalContainer(container ExternalContainer) error
@@ -46,6 +63,7 @@ type Context interface {
 	AttachNetwork(network Network, instance string) (NetworkInterface, error)
 	DetachNetwork(network Network, instance string) error
 	GetAttachedNetworks() (Networks, error)
+	SetupUnderlay(f *framework.Framework, underlay Underlay) error
 
 	AddCleanUpFn(func() error)
 }
@@ -164,13 +182,14 @@ func (n NetworkInterface) GetMAC() string {
 }
 
 type ExternalContainer struct {
-	Name    string
-	Image   string
-	Network Network
-	Args    []string
-	ExtPort uint16
-	IPv4    string
-	IPv6    string
+	Name       string
+	Image      string
+	Network    Network
+	Entrypoint string
+	Args       []string
+	ExtPort    uint16
+	IPv4       string
+	IPv6       string
 }
 
 func (ec ExternalContainer) GetName() string {

test/e2e/infraprovider/providers/kind/kind.go

@@ -13,10 +13,12 @@ import (
 
 	"github.com/onsi/ginkgo/v2"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/containerengine"
+	"github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/images"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/api"
 	"github.com/ovn-org/ovn-kubernetes/test/e2e/infraprovider/portalloc"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/kubernetes/test/e2e/framework"
 	utilnet "k8s.io/utils/net"
@@ -145,6 +147,9 @@ func (c *contextKind) createExternalContainer(container api.ExternalContainer) (
 		return container, fmt.Errorf("container %s already exists", container.Name)
 	}
 	cmd := []string{"run", "-itd", "--privileged", "--name", container.Name, "--network", container.Network.Name(), "--hostname", container.Name}
+	if container.Entrypoint != "" {
+		cmd = append(cmd, "--entrypoint", container.Entrypoint)
+	}
 	cmd = append(cmd, container.Image)
 	if len(container.Args) > 0 {
 		cmd = append(cmd, container.Args...)
@@ -359,6 +364,83 @@ func (c *contextKind) getAttachedNetworks() (api.Networks, error) {
 	return attachedNetworks, nil
 }
 
+func (c *contextKind) SetupUnderlay(f *framework.Framework, underlay api.Underlay) error {
+	if underlay.LogicalNetworkName == "" {
+		return fmt.Errorf("underlay logical network name must be set")
+	}
+
+	if underlay.PhysicalNetworkName == "" {
+		underlay.PhysicalNetworkName = "underlay"
+	}
+
+	if underlay.BridgeName == "" {
+		underlay.BridgeName = secondaryBridge
+	}
+
+	const (
+		ovsKubeNodeLabel = "app=ovnkube-node"
+	)
+
+	ovsPodList, err := f.ClientSet.CoreV1().Pods(deploymentconfig.Get().OVNKubernetesNamespace()).List(
+		context.Background(),
+		metav1.ListOptions{LabelSelector: ovsKubeNodeLabel},
+	)
+	if err != nil {
+		return fmt.Errorf("failed to list OVS pods with label %q at namespace %q: %w", ovsKubeNodeLabel, deploymentconfig.Get().OVNKubernetesNamespace(), err)
+	}
+
+	if len(ovsPodList.Items) == 0 {
+		return fmt.Errorf("no pods with label %q in namespace %q", ovsKubeNodeLabel, deploymentconfig.Get().OVNKubernetesNamespace())
+	}
+	for _, ovsPod := range ovsPodList.Items {
+		if underlay.BridgeName != deploymentconfig.Get().ExternalBridgeName() {
+			underlayInterface, err := getNetworkInterface(ovsPod.Spec.NodeName, underlay.PhysicalNetworkName)
+			if err != nil {
+				return fmt.Errorf("failed to get underlay interface for network %s on node %s: %w", underlay.PhysicalNetworkName, ovsPod.Spec.NodeName, err)
+			}
+			c.AddCleanUpFn(func() error {
+				if err := removeOVSBridge(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName); err != nil {
+					return fmt.Errorf("failed to remove OVS bridge %s for pod %s/%s during cleanup: %w", underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+				}
+				return nil
+			})
+			if err := ensureOVSBridge(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName); err != nil {
+				return fmt.Errorf("failed to add OVS bridge %s for pod %s/%s: %w", underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+			}
+
+			if err := ovsAttachPortToBridge(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName, underlayInterface.InfName); err != nil {
+				return fmt.Errorf("failed to attach port %s to bridge %s for pod %s/%s: %w", underlayInterface.InfName, underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+			}
+			if underlay.VlanID > 0 {
+				if err := ovsEnableVLANAccessPort(ovsPod.Namespace, ovsPod.Name, underlay.BridgeName, underlayInterface.InfName, underlay.VlanID); err != nil {
+					return fmt.Errorf("failed to enable VLAN %d on port %s for bridge %s for pod %s/%s: %w", underlay.VlanID, underlayInterface.InfName, underlay.BridgeName, ovsPod.Namespace, ovsPod.Name, err)
+				}
+			}
+		}
+		c.AddCleanUpFn(func() error {
+			if err := configureBridgeMappings(
+				ovsPod.Namespace,
+				ovsPod.Name,
+				defaultNetworkBridgeMapping(),
+			); err != nil {
+				return fmt.Errorf("failed to restore default bridge mappings for pod %s/%s during cleanup: %w", ovsPod.Namespace, ovsPod.Name, err)
+			}
+			return nil
+		})
+
+		if err := configureBridgeMappings(
+			ovsPod.Namespace,
+			ovsPod.Name,
+			defaultNetworkBridgeMapping(),
+			bridgeMapping(underlay.LogicalNetworkName, underlay.BridgeName),
+		); err != nil {
+			return fmt.Errorf("failed to configure bridge mappings for pod %s/%s for logical network %s to bridge %s: %w", ovsPod.Namespace, ovsPod.Name, underlay.LogicalNetworkName, underlay.BridgeName, err)
+		}
+	}
+	return nil
+
+}
+
 func (c *contextKind) AddCleanUpFn(cleanUpFn func() error) {
 	c.Lock()
 	defer c.Unlock()

test/e2e/infraprovider/providers/kind/ovs.go

@@ -0,0 +1,93 @@
+package kind
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/ovn-org/ovn-kubernetes/test/e2e/deploymentconfig"
+
+	e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
+)
+
+const (
+	secondaryBridge = "ovsbr1"
+)
+
+func ensureOVSBridge(podNamespace, podName string, bridgeName string) error {
+	cmd := fmt.Sprintf("ovs-vsctl br-exists %[1]s || ovs-vsctl add-br %[1]s", bridgeName)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to add ovs bridge %q: %v", bridgeName, err)
+	}
+	return nil
+}
+
+func removeOVSBridge(podNamespace, podName string, bridgeName string) error {
+	cmd := fmt.Sprintf("if ovs-vsctl br-exists %[1]s; then ovs-vsctl del-br %[1]s; fi", bridgeName)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to remove ovs bridge %q: %v", bridgeName, err)
+	}
+	return nil
+}
+
+func ovsAttachPortToBridge(podNamespace, podName string, bridgeName string, portName string) error {
+	cmd := fmt.Sprintf("ovs-vsctl list port %[2]s || ovs-vsctl add-port %[1]s %[2]s", bridgeName, portName)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to addadd  port %s from OVS bridge %s: %v", portName, bridgeName, err)
+	}
+	return nil
+}
+
+func ovsEnableVLANAccessPort(podNamespace, podName string, bridgeName string, portName string, vlanID int) error {
+	cmd := fmt.Sprintf("ovs-vsctl set port %[1]s tag=%[2]d vlan_mode=access", portName, vlanID)
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to enable vlan access port %s from OVS bridge %s: %v", portName, bridgeName, err)
+	}
+	return nil
+}
+
+type BridgeMapping struct {
+	physnet   string
+	ovsBridge string
+}
+
+func (bm BridgeMapping) String() string {
+	return fmt.Sprintf("%s:%s", bm.physnet, bm.ovsBridge)
+}
+
+type BridgeMappings []BridgeMapping
+
+func (bms BridgeMappings) String() string {
+	return strings.Join(Map(bms, func(bm BridgeMapping) string { return bm.String() }), ",")
+}
+
+func Map[T, V any](items []T, fn func(T) V) []V {
+	result := make([]V, len(items))
+	for i, t := range items {
+		result[i] = fn(t)
+	}
+	return result
+}
+
+func configureBridgeMappings(podNamespace, podName string, mappings ...BridgeMapping) error {
+	mappingsString := fmt.Sprintf("external_ids:ovn-bridge-mappings=%s", BridgeMappings(mappings).String())
+	cmd := strings.Join([]string{"ovs-vsctl", "set", "open", ".", mappingsString}, " ")
+	if _, err := e2epodoutput.RunHostCmdWithRetries(podNamespace, podName, cmd, time.Second, time.Second*5); err != nil {
+		return fmt.Errorf("failed to configure bridge mappings %q: %v", mappingsString, err)
+	}
+	return nil
+}
+
+func defaultNetworkBridgeMapping() BridgeMapping {
+	return BridgeMapping{
+		physnet:   "physnet",
+		ovsBridge: deploymentconfig.Get().ExternalBridgeName(),
+	}
+}
+
+func bridgeMapping(physnet, ovsBridge string) BridgeMapping {
+	return BridgeMapping{
+		physnet:   physnet,
+		ovsBridge: ovsBridge,
+	}
+}