Merge pull request #4847 from qinqon/kv-live-migration-cooked-ras

kubevirt, l2, udpn: send unsolicited router advertisement after live migration to reconcile ipv6 default gw

Author: tssurya

go-controller/go.mod

@@ -29,6 +29,7 @@ require (
 	github.com/k8snetworkplumbingwg/sriovnet v1.2.1-0.20230427090635-4929697df2dc
 	github.com/mdlayher/arp v0.0.0-20220512170110-6706a2966875
 	github.com/mdlayher/ndp v1.0.1
+	github.com/mdlayher/socket v0.2.1
 	github.com/metallb/frr-k8s v0.0.15
 	github.com/miekg/dns v1.1.31
 	github.com/mitchellh/copystructure v1.2.0
@@ -103,7 +104,6 @@ require (
 	github.com/mailru/easyjson v0.7.7 // indirect
 	github.com/mdlayher/ethernet v0.0.0-20220221185849-529eae5b6118 // indirect
 	github.com/mdlayher/packet v1.0.0 // indirect
-	github.com/mdlayher/socket v0.2.1 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
 	github.com/moby/spdystream v0.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect

go-controller/pkg/kubevirt/pod.go

@@ -22,8 +22,29 @@ import (
 	logicalswitchmanager "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/ovn/logical_switch_manager"
 	ovntypes "github.com/ovn-org/ovn-kubernetes/go-controller/pkg/types"
 	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util"
+	"github.com/ovn-org/ovn-kubernetes/go-controller/pkg/util/ndp"
 )
 
+// DefaultGatewayReconciler is responsible for reconciling the default gateway
+// configuration of a virtual machine's network interface after a live migration.
+// It supports both IPv4 and IPv6 configurations.
+type DefaultGatewayReconciler struct {
+	watchFactory  *factory.WatchFactory
+	netInfo       util.NetInfo
+	interfaceName string
+}
+
+// NewDefaultGatewayReconciler creates a new instance of DefaultGatewayReconciler.
+// It takes a WatchFactory for managing resource watches, a NetInfo object for network information,
+// and the name of the network interface to send ARPs or RAs as parameters.
+func NewDefaultGatewayReconciler(watchFactory *factory.WatchFactory, netInfo util.NetInfo, interfaceName string) *DefaultGatewayReconciler {
+	return &DefaultGatewayReconciler{
+		watchFactory:  watchFactory,
+		netInfo:       netInfo,
+		interfaceName: interfaceName,
+	}
+}
+
 // IsPodLiveMigratable will return true if the pod belongs
 // to kubevirt and should use the live migration features
 func IsPodLiveMigratable(pod *corev1.Pod) bool {
@@ -482,17 +503,20 @@ func DiscoverLiveMigrationStatus(client *factory.WatchFactory, pod *corev1.Pod)
 	return &status, nil
 }
 
-func ReconcileIPv4DefaultGatewayAfterLiveMigration(watchFactory *factory.WatchFactory, netInfo util.NetInfo, liveMigrationStatus *LiveMigrationStatus, interfaceName string) error {
+// ReconcileIPv4AfterLiveMigration will send a GARP after live migration
+// to update the default gw mac address to the node where the VM is running
+// now.
+func (r *DefaultGatewayReconciler) ReconcileIPv4AfterLiveMigration(liveMigrationStatus *LiveMigrationStatus) error {
 	if liveMigrationStatus.State != LiveMigrationTargetDomainReady {
 		return nil
 	}
 
-	targetNode, err := watchFactory.GetNode(liveMigrationStatus.TargetPod.Spec.NodeName)
+	targetNode, err := r.watchFactory.GetNode(liveMigrationStatus.TargetPod.Spec.NodeName)
 	if err != nil {
 		return err
 	}
 
-	lrpJoinAddress, err := util.ParseNodeGatewayRouterJoinNetwork(targetNode, netInfo.GetNetworkName())
+	lrpJoinAddress, err := util.ParseNodeGatewayRouterJoinNetwork(targetNode, r.netInfo.GetNetworkName())
 	if err != nil {
 		return err
 	}
@@ -503,16 +527,104 @@ func ReconcileIPv4DefaultGatewayAfterLiveMigration(watchFactory *factory.WatchFa
 	}
 
 	lrpMAC := util.IPAddrToHWAddr(lrpJoinIPv4)
-	for _, subnet := range netInfo.Subnets() {
+	for _, subnet := range r.netInfo.Subnets() {
 		gwIP := util.GetNodeGatewayIfAddr(subnet.CIDR).IP.To4()
 		if gwIP == nil {
 			continue
 		}
 		garp := util.GARP{IP: gwIP, MAC: &lrpMAC}
-		if err := util.BroadcastGARP(interfaceName, garp); err != nil {
+		if err := util.BroadcastGARP(r.interfaceName, garp); err != nil {
 			return err
 		}
 	}
-
 	return nil
 }
+
+// ReconcileIPv6AfterLiveMigration will do two things at VM's:
+// - Remove ipv6 default gw path from VM's node before live migration
+// - Add ipv6 default gw path from VM's node after live migration
+// This is done by sending a pair of unsolicited RA's one with lifetime=0
+// (to remove the gateway path) another with lifetime=max to add the new
+// default gateway path
+func (r *DefaultGatewayReconciler) ReconcileIPv6AfterLiveMigration(liveMigration *LiveMigrationStatus) error {
+	if !liveMigration.IsTargetDomainReady() {
+		return nil
+	}
+	nodes, err := r.watchFactory.GetNodes()
+	if err != nil {
+		return err
+	}
+
+	targetPod := liveMigration.TargetPod
+	if len(r.netInfo.GetNADs()) != 1 {
+		return fmt.Errorf("expected only one nad for network %q, got %d", r.netInfo.GetNetworkName(), len(r.netInfo.GetNADs()))
+	}
+
+	targetPodAnnotation, err := util.UnmarshalPodAnnotation(targetPod.Annotations, r.netInfo.GetNADs()[0])
+	if err != nil {
+		return ovntypes.NewSuppressedError(fmt.Errorf("failed parsing ovn pod annotation for pod '%s/%s' and network %q: %w", targetPod.Namespace, targetPod.Name, r.netInfo.GetNetworkName(), err))
+	}
+
+	destinationIP, err := util.MatchFirstIPNetFamily(true /* ipv6 */, targetPodAnnotation.IPs)
+	if err != nil {
+		return err
+	}
+	destinationMAC := targetPodAnnotation.MAC
+
+	ras := make([]ndp.RouterAdvertisement, 0, len(nodes))
+	for _, node := range nodes {
+		if node.Name == liveMigration.TargetPod.Spec.NodeName {
+			// skip the target node since this is the proper gateway
+			continue
+		}
+		nodeJoinAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(node, r.netInfo.GetNetworkName())
+		if err != nil {
+			return ovntypes.NewSuppressedError(fmt.Errorf("failed parsing join addresss from node %q and network %q to reconcile ipv6 gateway: %w", node.Name, r.netInfo.GetNetworkName(), err))
+		}
+		// During upgrades, nftables blocks Router Advertisements (RAs) from other nodes.
+		// However, Virtual Machines (VMs) may still retain old default gateway paths.
+		// To address this, we create a new Router Advertisement with a lifetime of 0
+		// to signal the removal of the old default gateway.
+		// NOTE: This is a workaround for the issue and may not be needed in the future, after
+		//       upgrading to a version that supports the new behavior.
+		ras = append(ras, newRouterAdvertisementFromJoinIPAndLifetime(nodeJoinAddrs[0].IP, destinationMAC, destinationIP.IP, 0))
+	}
+	targetNode, err := r.watchFactory.GetNode(liveMigration.TargetPod.Spec.NodeName)
+	if err != nil {
+		return fmt.Errorf("failed fetching node %q to reconcile ipv6 gateway: %w", liveMigration.TargetPod.Spec.NodeName, err)
+	}
+	targetNodeJoinAddrs, err := util.ParseNodeGatewayRouterJoinAddrs(targetNode, r.netInfo.GetNetworkName())
+	if err != nil {
+		return ovntypes.NewSuppressedError(fmt.Errorf("failed parsing join addresss from live migration target node %q and network %q to reconcile ipv6 gateway: %w", targetNode.Name, r.netInfo.GetNetworkName(), err))
+	}
+	ras = append(ras, newRouterAdvertisementFromJoinIPAndLifetime(targetNodeJoinAddrs[0].IP, destinationMAC, destinationIP.IP, 65535))
+	return ndp.SendRouterAdvertisements(r.interfaceName, ras...)
+}
+
+// newRouterAdvertisementFromJoinIPAndLifetime creates a new Router Advertisement (RA) message
+// using the provided join IP address, destination MAC, destination IP, and lifetime.
+//
+// This function performs the following:
+// - Derives the source MAC address from the given IP using util.IPAddrToHWAddr.
+// - Calculates the link-local address (LLA) from the source MAC using util.HWAddrToIPv6LLA.
+// - Configures the destination IP and MAC address to use the provided values.
+// - Sets the RA message's lifetime to the specified value.
+//
+// Parameters:
+// - ip: The join IP address used to derive the source MAC and LLA.
+// - destinationMAC: The MAC address to which the RA message will be sent.
+// - destinationIP: The IP address to which the RA message will be sent.
+// - lifetime: The lifetime value for the RA message, in seconds.
+//
+// Returns:
+// - An ndp.RouterAdvertisement object configured with the calculated source MAC, LLA, and the provided destination MAC, IP, and lifetime.
+func newRouterAdvertisementFromJoinIPAndLifetime(ip net.IP, destinationMAC net.HardwareAddr, destinationIP net.IP, lifetime uint16) ndp.RouterAdvertisement {
+	sourceMAC := util.IPAddrToHWAddr(ip)
+	return ndp.RouterAdvertisement{
+		SourceMAC:      sourceMAC,
+		SourceIP:       util.HWAddrToIPv6LLA(sourceMAC),
+		DestinationMAC: destinationMAC,
+		DestinationIP:  destinationIP,
+		Lifetime:       lifetime,
+	}
+}

go-controller/pkg/ovn/secondary_layer2_network_controller.go

@@ -293,6 +293,9 @@ type SecondaryLayer2NetworkController struct {
 
 	// EgressIP controller utilized only to initialize a network with OVN polices to support EgressIP functionality.
 	eIPController *EgressIPController
+
+	// reconcile the virtual machine default gateway sending GARPs and RAs
+	defaultGatewayReconciler *kubevirt.DefaultGatewayReconciler
 }
 
 // NewSecondaryLayer2NetworkController create a new OVN controller for the given secondary layer2 nad
@@ -363,6 +366,7 @@ func NewSecondaryLayer2NetworkController(
 		if err != nil {
 			return nil, fmt.Errorf("unable to create new service controller while creating new layer2 network controller: %w", err)
 		}
+		oc.defaultGatewayReconciler = kubevirt.NewDefaultGatewayReconciler(oc.watchFactory, oc.GetNetInfo(), util.GetNetworkScopedK8sMgmtHostIntfName(uint(oc.GetNetworkID())))
 	}
 
 	if oc.allocatesPodAnnotation() {
@@ -908,15 +912,20 @@ func (oc *SecondaryLayer2NetworkController) updateLocalPodEvent(pod *corev1.Pod)
 }
 
 func (oc *SecondaryLayer2NetworkController) reconcileLiveMigrationTargetZone(kubevirtLiveMigrationStatus *kubevirt.LiveMigrationStatus) error {
-	// Only primary networks has a gateway to reconcile
-	if !oc.IsPrimaryNetwork() {
+	if oc.defaultGatewayReconciler == nil {
 		return nil
 	}
-	mgmtInterfaceName := util.GetNetworkScopedK8sMgmtHostIntfName(uint(oc.GetNetworkID()))
-
-	if hasIPv4Subnet, _ := oc.IPMode(); hasIPv4Subnet {
-		if err := kubevirt.ReconcileIPv4DefaultGatewayAfterLiveMigration(oc.watchFactory, oc.GetNetInfo(), kubevirtLiveMigrationStatus, mgmtInterfaceName); err != nil {
-			return err
+	hasIPv4Subnet, hasIPv6Subnet := oc.IPMode()
+	if hasIPv4Subnet {
+		if err := oc.defaultGatewayReconciler.ReconcileIPv4AfterLiveMigration(kubevirtLiveMigrationStatus); err != nil {
+			return fmt.Errorf("failed reconciling IPv4 default gw after live migration at target pod '%s/%s': %w",
+				kubevirtLiveMigrationStatus.TargetPod.Namespace, kubevirtLiveMigrationStatus.TargetPod.Name, err)
+		}
+	}
+	if hasIPv6Subnet {
+		if err := oc.defaultGatewayReconciler.ReconcileIPv6AfterLiveMigration(kubevirtLiveMigrationStatus); err != nil {
+			return fmt.Errorf("failed reconciling IPv6 default gw after live migration at target pod '%s/%s': %w",
+				kubevirtLiveMigrationStatus.TargetPod.Namespace, kubevirtLiveMigrationStatus.TargetPod.Name, err)
 		}
 	}
 	return nil

go-controller/pkg/util/ndp/ra.go

@@ -0,0 +1,125 @@
+package ndp
+
+import (
+	"fmt"
+	"net"
+	"syscall"
+
+	"github.com/google/gopacket"
+	"github.com/google/gopacket/layers"
+	"github.com/mdlayher/socket"
+	"golang.org/x/sys/unix"
+)
+
+// RouterAdvertisement with mac, ips and lifetime field to send
+type RouterAdvertisement struct {
+	SourceMAC, DestinationMAC net.HardwareAddr
+	SourceIP, DestinationIP   net.IP
+	Lifetime                  uint16
+}
+
+// SendRouterAdvertisements sends one or more Router Advertisements (RAs) on the specified network interface.
+// This function requires raw socket capabilities because the source MAC and IP addresses in the RAs
+// are not the ones from the interface used to send the packets.
+//
+// Parameters:
+// - interfaceName: The name of the network interface to send the RAs on.
+// - ras: A variadic list of RouterAdvertisement objects containing the details of each RA to be sent.
+//
+// Returns:
+// - error: An error object if an error occurs, otherwise nil.
+//
+// The function performs the following steps:
+// 1. Retrieves the network interface by name.
+// 2. Creates a raw socket for sending packets.
+// 3. Serializes each Router Advertisement into a byte slice.
+// 4. Sends the serialized RAs using the raw socket.
+func SendRouterAdvertisements(interfaceName string, ras ...RouterAdvertisement) error {
+	iface, err := net.InterfaceByName(interfaceName)
+	if err != nil {
+		return fmt.Errorf("failed to find interface %s: %w", interfaceName, err)
+	}
+	c, err := socket.Socket(syscall.AF_PACKET, syscall.SOCK_RAW, syscall.ETH_P_ALL, "ra", nil)
+	if err != nil {
+		return fmt.Errorf("failed to create raw socket to send unsolicited RAs: %w", err)
+	}
+	defer c.Close()
+
+	serializedRAs := [][]byte{}
+	for _, ra := range ras {
+		serializeBuffer := gopacket.NewSerializeBuffer()
+
+		// Create the Ethernet layer with destination and source MAC addresses.
+		ethernetLayer := layers.Ethernet{
+			DstMAC:       ra.DestinationMAC,
+			SrcMAC:       ra.SourceMAC,
+			EthernetType: layers.EthernetTypeIPv6,
+		}
+
+		// Create the IPv6 layer with source and destination IP addresses.
+		ip6Layer := layers.IPv6{
+			Version:    6,
+			NextHeader: layers.IPProtocolICMPv6,
+			HopLimit:   255,
+			SrcIP:      ra.SourceIP,
+			DstIP:      ra.DestinationIP,
+		}
+
+		// Create the ICMPv6 layer for the Router Advertisement.
+		icmp6Layer := layers.ICMPv6{
+			TypeCode: layers.CreateICMPv6TypeCode(layers.ICMPv6TypeRouterAdvertisement, 0),
+		}
+		if err := icmp6Layer.SetNetworkLayerForChecksum(&ip6Layer); err != nil {
+			return err
+		}
+
+		// https://datatracker.ietf.org/doc/html/rfc4861#section-4.2
+		// Managed address configuration flag.
+		managedAddressFlag := uint8(0x80)
+
+		// https://datatracker.ietf.org/doc/html/rfc4191#section-2.2
+		// Prf (Default Router Preference)
+		//   2-bit signed integer.  Indicates whether to prefer this
+		//    router over other default routers.  If the Router Lifetime
+		//    is zero, the preference value MUST be set to (00) by the
+		//    sender and MUST be ignored by the receiver.  If the Reserved
+		//    (10) value is received, the receiver MUST treat the value as
+		//    if it were (00).
+		defaultRoutePreferenceFlag := uint8(0x08)
+		if ra.Lifetime == 0 {
+			defaultRoutePreferenceFlag = uint8(0x00)
+		}
+
+		// Create the ICMPv6 Router Advertisement layer.
+		raLayer := layers.ICMPv6RouterAdvertisement{
+			HopLimit:       255,
+			Flags:          managedAddressFlag | defaultRoutePreferenceFlag,
+			RouterLifetime: ra.Lifetime,
+			ReachableTime:  0,
+			RetransTimer:   0,
+			Options: layers.ICMPv6Options{{
+				Type: layers.ICMPv6OptSourceAddress,
+				Data: ra.SourceMAC,
+			}},
+		}
+
+		// Serialize the layers into a byte slice.
+		if err := gopacket.SerializeLayers(serializeBuffer, gopacket.SerializeOptions{ComputeChecksums: true, FixLengths: true},
+			&ethernetLayer,
+			&ip6Layer,
+			&icmp6Layer,
+			&raLayer,
+		); err != nil {
+			return err
+		}
+		serializedRAs = append(serializedRAs, serializeBuffer.Bytes())
+	}
+
+	// Send each serialized Router Advertisement using the raw socket.
+	for _, serializedRA := range serializedRAs {
+		if err := c.Sendto(serializedRA, &unix.SockaddrLinklayer{Ifindex: iface.Index}, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}