[draft] [v3.29] fix for issues on ipv6 enabled clusters

calico-vpp-agent/cni/cni_server.go

@@ -538,10 +538,17 @@ func (s *Server) createRedirectToHostRules() (uint32, error) {
 		return types.InvalidID, fmt.Errorf("no main interface found")
 	}
 	for _, rule := range config.GetCalicoVppInitialConfig().RedirectToHostRules {
+		mainInterfaceAddress := mainInterface.GetAddress(vpplink.IPFamilyFromIP(rule.IP))
+		if mainInterfaceAddress == nil {
+			return types.InvalidID, fmt.Errorf("error installing rule %v no address found on uplink", rule)
+		}
 		err = s.vpp.AddSessionRedirect(&types.SessionRedirect{
-			FiveTuple:  types.NewDst3Tuple(rule.Proto, net.ParseIP(rule.IP), rule.Port),
+			FiveTuple:  types.NewDst3Tuple(rule.Proto, rule.IP, rule.Port),
 			TableIndex: index,
-		}, &types.RoutePath{Gw: config.VppHostPuntFakeGatewayAddress, SwIfIndex: mainInterface.TapSwIfIndex})
+		}, &types.RoutePath{
+			Gw:        mainInterfaceAddress.IP,
+			SwIfIndex: mainInterface.TapSwIfIndex,
+		})
 		if err != nil {
 			return types.InvalidID, err
 		}

calico-vpp-agent/routing/bgp_watcher.go

@@ -543,10 +543,14 @@ func (s *Server) WatchBGPPath(t *tomb.Tomb) error {
 				peer := localPeer.Peer
 				filters := localPeer.BGPFilterNames
 				// create a neighbor set to apply filter only on specific peer using a global policy
+				prefixLen := "/32"
+				if ip := net.ParseIP(peer.Conf.NeighborAddress); ip != nil && ip.To4() == nil {
+					prefixLen = "/128"
+				}
 				neighborSet := &bgpapi.DefinedSet{
 					Name:        peer.Conf.NeighborAddress + "neighbor",
 					DefinedType: bgpapi.DefinedType_NEIGHBOR,
-					List:        []string{peer.Conf.NeighborAddress + "/32"},
+					List:        []string{peer.Conf.NeighborAddress + prefixLen},
 				}
 				err := s.BGPServer.AddDefinedSet(context.Background(), &bgpapi.AddDefinedSetRequest{
 					DefinedSet: neighborSet,
@@ -580,9 +584,18 @@ func (s *Server) WatchBGPPath(t *tomb.Tomb) error {
 				if err != nil {
 					return errors.Wrapf(err, "error cleaning peer filters up")
 				}
-				err = s.BGPServer.DeleteDefinedSet(context.Background(), &bgpapi.DeleteDefinedSetRequest{DefinedSet: s.bgpPeers[addr].NeighborSet, All: true})
-				if err != nil {
-					return errors.Wrapf(err, "error deleting prefix set")
+				if s.bgpPeers[addr] == nil {
+					s.log.Warnf("Trying to delete unknown BGP peer %s", addr)
+				} else if s.bgpPeers[addr].NeighborSet == nil {
+					s.log.Warnf("Trying to delete BGP peer %s with empty NeighborSet", addr)
+				} else {
+					err = s.BGPServer.DeleteDefinedSet(context.Background(), &bgpapi.DeleteDefinedSetRequest{
+						DefinedSet: s.bgpPeers[addr].NeighborSet,
+						All:        true,
+					})
+					if err != nil {
+						return errors.Wrapf(err, "error deleting prefix set")
+					}
 				}
 				err := s.BGPServer.DeletePeer(
 					context.Background(),

calico-vpp-agent/routing/routing_server.go

@@ -31,6 +31,7 @@ import (
 
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common"
 	"github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/watchers"
+	"github.com/projectcalico/vpp-dataplane/v3/config"
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 )
 
@@ -114,17 +115,28 @@ func (s *Server) ServeRouting(t *tomb.Tomb) (err error) {
 	}
 
 	for t.Alive() {
-		globalConfig, err := s.getGoBGPGlobalConfig()
+		nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec)
+		globalConfig, err := s.getGoBGPGlobalConfig(*config.BGPServerMode)
 		if err != nil {
 			return fmt.Errorf("cannot get global configuration: %v", err)
 		}
 
 		err = s.BGPServer.StartBgp(context.Background(), &bgpapi.StartBgpRequest{Global: globalConfig})
-		if err != nil {
+		if err != nil && *config.BGPServerMode == config.BGPServerModeDualStack && nodeIP4 != nil {
+			s.log.Warnf("Failed to start BGP server in dualStack mode: %v. Retrying with IPv4-only listener", err)
+			globalConfig, err = s.getGoBGPGlobalConfig(config.BGPServerModeV4Only)
+			if err != nil {
+				return errors.Wrap(err, "cannot get IPv4-only BGP configuration for fallback")
+			}
+			err = s.BGPServer.StartBgp(context.Background(), &bgpapi.StartBgpRequest{Global: globalConfig})
+			if err != nil {
+				return errors.Wrap(err, "failed to start BGP server after IPv4-only fallback")
+			}
+			s.log.Warn("BGP server started in degraded IPv4-only mode because IPv6 listener failed")
+		} else if err != nil {
 			return errors.Wrap(err, "failed to start BGP server")
 		}
 
-		nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec)
 		if nodeIP4 != nil {
 			err = s.initialPolicySetting(false /* isv6 */)
 			if err != nil {
@@ -176,7 +188,7 @@ func (s *Server) getLogSeverityScreen() string {
 	return s.BGPConf.LogSeverityScreen
 }
 
-func (s *Server) getGoBGPGlobalConfig() (*bgpapi.Global, error) {
+func (s *Server) getGoBGPGlobalConfig(mode config.BGPServerModeType) (*bgpapi.Global, error) {
 	var routerID string
 	listenAddresses := make([]string, 0)
 	asn := s.nodeBGPSpec.ASNumber
@@ -185,11 +197,25 @@ func (s *Server) getGoBGPGlobalConfig() (*bgpapi.Global, error) {
 	}
 
 	nodeIP4, nodeIP6 := common.GetBGPSpecAddresses(s.nodeBGPSpec)
-	if nodeIP6 != nil {
+	useIP4 := nodeIP4 != nil
+	useIP6 := nodeIP6 != nil
+
+	switch mode {
+	case config.BGPServerModeDualStack:
+	case config.BGPServerModeV4Only:
+		useIP6 = false
+		if !useIP4 {
+			return nil, fmt.Errorf("BGP server mode set to v4Only but no IPv4 node address configured")
+		}
+	default:
+		return nil, fmt.Errorf("unsupported BGP server mode %q", mode)
+	}
+
+	if useIP6 {
 		routerID = nodeIP6.String()
 		listenAddresses = append(listenAddresses, routerID)
 	}
-	if nodeIP4 != nil {
+	if useIP4 {
 		routerID = nodeIP4.String() // Override v6 ID if v4 is available
 		listenAddresses = append(listenAddresses, routerID)
 	}

config/config.go

@@ -64,14 +64,22 @@ const (
 	BaseVppSideHardwareAddress = "02:ca:11:c0:fd:00"
 )
 
+type BGPServerModeType string
+
+const (
+	BGPServerModeDualStack BGPServerModeType = "dualStack"
+	BGPServerModeV4Only    BGPServerModeType = "v4Only"
+)
+
 var (
 	// fake constants for place where we need a pointer to true or false
 	True  = true
 	False = false
 
-	NodeName    = RequiredStringEnvVar("NODENAME")
-	LogLevel    = EnvVar("CALICOVPP_LOG_LEVEL", logrus.InfoLevel, logrus.ParseLevel)
-	BGPLogLevel = EnvVar("CALICOVPP_BGP_LOG_LEVEL", apipb.SetLogLevelRequest_INFO, BGPLogLevelParse)
+	NodeName      = RequiredStringEnvVar("NODENAME")
+	LogLevel      = EnvVar("CALICOVPP_LOG_LEVEL", logrus.InfoLevel, logrus.ParseLevel)
+	BGPLogLevel   = EnvVar("CALICOVPP_BGP_LOG_LEVEL", apipb.SetLogLevelRequest_INFO, BGPLogLevelParse)
+	BGPServerMode = EnvVar("CALICOVPP_BGP_SERVER_MODE", BGPServerModeDualStack, BGPServerModeParse)
 
 	ServiceCIDRs                     = PrefixListEnvVar("SERVICE_PREFIX")
 	IPSecIkev2Psk                    = StringEnvVar("CALICOVPP_IPSEC_IKEV2_PSK", "")
@@ -115,6 +123,8 @@ var (
 	/* Run this before getLinuxConfig() in case this is a script
 	 * that's responsible for creating the interface */
 	HookScriptBeforeIfRead = StringEnvVar("CALICOVPP_HOOK_BEFORE_IF_READ", DefaultHookScript) // InitScriptTemplate
+	/* Bash script template run to capture host udev properties before driver unbind */
+	HookScriptCaptureHostUdevProps = StringEnvVar("CALICOVPP_HOOK_CAPTURE_HOST_UDEV_PROPS", DefaultHookScript)
 	/* Bash script template run just after getting config
 	   from $CALICOVPP_INTERFACE & before starting VPP */
 	HookScriptBeforeVppRun = StringEnvVar("CALICOVPP_HOOK_BEFORE_VPP_RUN", DefaultHookScript) // InitPostIfScriptTemplate
@@ -127,17 +137,24 @@ var (
 
 	AllHooks = []*string{
 		HookScriptBeforeIfRead,
+		HookScriptCaptureHostUdevProps,
 		HookScriptBeforeVppRun,
 		HookScriptVppRunning,
 		HookScriptVppDoneOk,
 		HookScriptVppErrored,
 	}
 
-	Info = &VppManagerInfo{}
+	Info = &VppManagerInfo{
+		UplinkStatuses: make(map[string]UplinkStatus),
+		PhysicalNets:   make(map[string]PhysicalNetwork),
+	}
 
-	// VppHostPuntFakeGatewayAddress is the fake gateway we use with a static neighbor
-	// in the punt table to route punted packets to the host
-	VppHostPuntFakeGatewayAddress = net.ParseIP("169.254.0.1")
+	// VppsideTap0Address is the IP address we add to the tap0
+	// so that it can receive ipv4 packets
+	VppsideTap0Address = PrefixEnvVar(
+		"CALICOVPP_TAP0_ADDR",
+		MustParseCIDR("169.254.0.1/32"),
+	)
 )
 
 func RunHook(hookScript *string, hookName string, params *VppManagerParams, log *logrus.Logger) {
@@ -150,7 +167,7 @@ func RunHook(hookScript *string, hookName string, params *VppManagerParams, log
 		return
 	}
 
-	cmd := exec.Command("/bin/bash", "-c", template, hookName)
+	cmd := exec.Command("/bin/bash", "-c", template, hookName, params.UplinksSpecs[0].InterfaceName)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
@@ -271,7 +288,7 @@ func (u *UplinkInterfaceSpec) String() string {
 
 type RedirectToHostRulesConfigType struct {
 	Port uint16 `json:"port,omitempty"`
-	IP   string `json:"ip,omitempty"`
+	IP   net.IP `json:"ip,omitempty"`
 	/* "tcp", "udp",... */
 	Proto types.IPProto `json:"proto,omitempty"`
 }
@@ -567,6 +584,17 @@ type UplinkStatus struct {
 	// FakeNextHopIP6 is the computed next hop for v6 routes added
 	// in linux to (ServiceCIDR, podCIDR, etc...) towards this interface
 	FakeNextHopIP6 net.IP
+
+	UplinkAddresses []*net.IPNet
+}
+
+func (uplinkStatus *UplinkStatus) GetAddress(ipFamily vpplink.IPFamily) *net.IPNet {
+	for _, addr := range uplinkStatus.UplinkAddresses {
+		if vpplink.IPFamilyFromIPNet(addr) == ipFamily {
+			return addr
+		}
+	}
+	return nil
 }
 
 type PhysicalNetwork struct {

config/config_parse.go

@@ -162,7 +162,18 @@ func prefixParser(value string) (net.IPNet, error) {
 func RequiredPrefixEnvVar(varName string) *net.IPNet {
 	return RequiredEnvVar(varName, net.IPNet{}, prefixParser)
 }
-func PrefixEnvVar(varName string) *net.IPNet { return EnvVar(varName, net.IPNet{}, prefixParser) }
+
+func PrefixEnvVar(varName string, defaultValue *net.IPNet) *net.IPNet {
+	return EnvVar(varName, *defaultValue, prefixParser)
+}
+
+func MustParseCIDR(str string) *net.IPNet {
+	_, cidr, err := net.ParseCIDR(str)
+	if err != nil {
+		logrus.Fatalf("error parsing %s as cidr %v", str, err)
+	}
+	return cidr
+}
 
 func prefixListParser(value string) ([]*net.IPNet, error) {
 	chunks := strings.Split(value, ",")
@@ -257,3 +268,14 @@ func BGPLogLevelParse(lvl string) (apipb.SetLogLevelRequest_Level, error) {
 	var l apipb.SetLogLevelRequest_Level
 	return l, fmt.Errorf("not a valid logrus Level: %q", lvl)
 }
+
+func BGPServerModeParse(mode string) (BGPServerModeType, error) {
+	switch strings.ToLower(mode) {
+	case strings.ToLower(string(BGPServerModeDualStack)):
+		return BGPServerModeDualStack, nil
+	case "v4only":
+		return BGPServerModeV4Only, nil
+	}
+
+	return BGPServerModeDualStack, fmt.Errorf("not a valid BGP server mode: %q", mode)
+}