From 2e2237bd05aabb7cf6a7ac2704db3a94de798098 Mon Sep 17 00:00:00 2001 From: Nathan Skrzypczak Date: Fri, 5 Dec 2025 18:56:13 +0100 Subject: [PATCH] Vpp-manager refactoring Signed-off-by: Nathan Skrzypczak --- config/config.go | 58 +- vpp-manager/hooks/network_manager_hook.go | 32 +- vpp-manager/main.go | 44 +- vpp-manager/startup/interface_config.go | 103 ---- vpp-manager/startup/startup.go | 103 ++-- vpp-manager/uplink/af_packet.go | 39 +- vpp-manager/uplink/af_xdp.go | 55 +- vpp-manager/uplink/avf.go | 51 +- vpp-manager/uplink/common.go | 64 +-- vpp-manager/uplink/default.go | 39 +- vpp-manager/uplink/dpdk.go | 51 +- vpp-manager/uplink/rdma.go | 31 +- vpp-manager/uplink/virtio.go | 31 +- vpp-manager/uplink/vmxnet3.go | 25 +- vpp-manager/utils/utils.go | 21 + vpp-manager/vpp_runner.go | 650 +++++++++------------- 16 files changed, 597 insertions(+), 800 deletions(-) diff --git a/config/config.go b/config/config.go index 53402910..dc6c74ac 100644 --- a/config/config.go +++ b/config/config.go @@ -162,13 +162,13 @@ func RunHook(hookScript *string, hookName string, params *VppManagerParams, log if *hookScript == "" { return } - template, err := TemplateScriptReplace(*hookScript, params, nil) + template, err := TemplateScriptReplace(*hookScript, params) if err != nil { log.Warnf("Running hook %s errored with %s", hookName, err) return } - cmd := exec.Command("/bin/bash", "-c", template, hookName, params.UplinksSpecs[0].InterfaceName) + cmd := exec.Command("/bin/bash", "-c", template, hookName, params.InterfacesById[0].Spec.InterfaceName) cmd.Stdout = os.Stdout cmd.Stderr = os.Stderr err = cmd.Run() @@ -428,8 +428,6 @@ type CalicoVppInitialConfigConfigType struct { //out of agent and vppmanager // CorePattern is the pattern to use for VPP corefiles. // Usually "/var/lib/vpp/vppcore.%e.%p" CorePattern string `json:"corePattern"` - ExtraAddrCount int `json:"extraAddrCount"` - IfConfigSavePath string `json:"ifConfigSavePath"` // DefaultGWs Comma separated list of IPs to be // configured in VPP as default GW DefaultGWs string `json:"defaultGWs"` @@ -619,9 +617,35 @@ const ( VfioUnsafeNoIommuModeDISABLED UnsafeNoIommuMode = "disabled" ) +type UplinkDriver interface { + PreconfigureLinux() error + CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *UplinkInterfaceSpec) error + RestoreLinux() + IsSupported(warn bool) bool + GetName() string + UpdateVppConfigFile(template string) string + GetDefaultRxMode() types.RxMode +} + +type VppManagerInterface struct { + Spec UplinkInterfaceSpec + State *LinuxInterfaceState + Driver UplinkDriver +} + +func (p *VppManagerParams) AllInterfacesPhysical() bool { + for _, intf := range p.Interfaces { + if intf.State.IsTunTap || intf.State.IsVeth { + return false + } + } + return true +} + type VppManagerParams struct { - UplinksSpecs []UplinkInterfaceSpec - /* Capabilities */ + Interfaces map[string]*VppManagerInterface + InterfacesById []*VppManagerInterface + // Capabilities LoadedDrivers map[string]bool KernelVersion *KernelVersion AvailableHugePages int @@ -744,23 +768,25 @@ func getCpusetCPU() (string, error) { return regexp.MustCompile("[,-]").Split(cpusetCPU, 2)[0], nil } -func TemplateScriptReplace(input string, params *VppManagerParams, conf []*LinuxInterfaceState) (template string, err error) { +func TemplateScriptReplace(input string, params *VppManagerParams) (template string, err error) { template = input - if conf != nil { - /* We might template scripts before reading interface conf */ - template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", conf[0].PciID) - for i, ifcConf := range conf { - template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(i)+"__", ifcConf.PciID) - } + if len(params.Interfaces) > 0 { + // We might template scripts before reading interface conf + template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", params.InterfacesById[0].State.PciID) + } + for idx, intf := range params.InterfacesById { + template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(idx)+"__", intf.State.PciID) } vppcpu, err := getCpusetCPU() if err != nil { return "", err } template = strings.ReplaceAll(template, "__CPUSET_CPUS_FIRST__", vppcpu) - template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.UplinksSpecs[0].InterfaceName) - for i, ifc := range params.UplinksSpecs { - template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", i)+"__", ifc.InterfaceName) + if len(params.Interfaces) > 0 { + template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.InterfacesById[0].Spec.InterfaceName) + } + for idx, intf := range params.InterfacesById { + template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", idx)+"__", intf.Spec.InterfaceName) } for key, value := range params.NodeAnnotations { template = strings.ReplaceAll(template, fmt.Sprintf("__NODE_ANNOTATION:%s__", key), value) diff --git a/vpp-manager/hooks/network_manager_hook.go b/vpp-manager/hooks/network_manager_hook.go index d198f756..7ee125e5 100644 --- a/vpp-manager/hooks/network_manager_hook.go +++ b/vpp-manager/hooks/network_manager_hook.go @@ -52,9 +52,9 @@ type SystemType struct { // NetworkManagerHook manages network configuration during VPP lifecycle type NetworkManagerHook struct { - interfaceNames []string - systemType SystemType - log *logrus.Logger + vppManagerParams *config.VppManagerParams + systemType SystemType + log *logrus.Logger } // chrootCommand creates a command that will be executed in the host namespace @@ -151,21 +151,16 @@ func (h *NetworkManagerHook) restartService(serviceName string) error { } // NewNetworkManagerHook creates a new NetworkManagerHook instance -func NewNetworkManagerHook(log *logrus.Logger) *NetworkManagerHook { +func NewNetworkManagerHook(log *logrus.Logger, vppManagerParams *config.VppManagerParams) *NetworkManagerHook { hook := &NetworkManagerHook{ - log: log, + log: log, + vppManagerParams: vppManagerParams, } hook.detectSystem() return hook } -// SetInterfaceNames updates the interface names of NetworkManagerHook instance -func (h *NetworkManagerHook) SetInterfaceNames(interfaceNames []string) { - h.interfaceNames = interfaceNames - h.log.Infof("NetworkManagerHook: Interface names updated to %v", interfaceNames) -} - // fixDNS modifies NetworkManager configuration to disable DNS management func (h *NetworkManagerHook) fixDNS() error { if !h.systemType.HasNetworkManager { @@ -500,7 +495,7 @@ func (h *NetworkManagerHook) beforeVppRun() error { } // Save network file for AWS systemd-networkd for each interface - for _, interfaceName := range h.interfaceNames { + for interfaceName := range h.vppManagerParams.Interfaces { err = h.saveNetworkFile(interfaceName) if err != nil { return err @@ -519,7 +514,7 @@ func (h *NetworkManagerHook) vppRunning() error { } // Tweak network file for AWS systemd-networkd for each interface - for _, interfaceName := range h.interfaceNames { + for interfaceName := range h.vppManagerParams.Interfaces { err = h.tweakNetworkFile(interfaceName) if err != nil { return err @@ -538,7 +533,7 @@ func (h *NetworkManagerHook) vppDoneOk() error { } // Remove the tweaked network file for AWS systemd-networkd for each interface - for _, interfaceName := range h.interfaceNames { + for interfaceName := range h.vppManagerParams.Interfaces { err = h.removeTweakedNetworkFile(interfaceName) if err != nil { return err @@ -566,7 +561,7 @@ func (h *NetworkManagerHook) Execute(hookPoint HookPoint) error { return nil } - h.log.Infof("NetworkManagerHook: Executing %s for interfaces %v", hookPoint, h.interfaceNames) + h.log.Infof("NetworkManagerHook: Executing %s for all interfaces", hookPoint) var err error switch hookPoint { @@ -616,11 +611,10 @@ func (h *NetworkManagerHook) useDefaultHookFallback(userHook *string) bool { // 2. If native hooks enabled -> run native Go hook // 3. If native hooks disabled & no user script -> fallback to default_hook.sh func (h *NetworkManagerHook) ExecuteWithUserScript(hookPoint HookPoint, - hookScript *string, - params *config.VppManagerParams) { + hookScript *string) { // Check if user script is configured (highest priority - overrides everything) if hookScript != nil && *hookScript != "" { - config.RunHook(hookScript, string(hookPoint), params, h.log) + config.RunHook(hookScript, string(hookPoint), h.vppManagerParams, h.log) return } @@ -636,6 +630,6 @@ func (h *NetworkManagerHook) ExecuteWithUserScript(hookPoint HookPoint, // Fallback to default_hook.sh when native hooks are disabled if h.useDefaultHookFallback(hookScript) { h.log.Infof("Native hooks disabled, falling back to default_hook.sh for %s", hookPoint) - config.RunHook(&config.DefaultHookScript, string(hookPoint), params, h.log) + config.RunHook(&config.DefaultHookScript, string(hookPoint), h.vppManagerParams, h.log) } } diff --git a/vpp-manager/main.go b/vpp-manager/main.go index 5058acfa..1446980b 100644 --- a/vpp-manager/main.go +++ b/vpp-manager/main.go @@ -165,8 +165,8 @@ func main() { /* Initialize native Go NewNetworkManagerHook with empty interface names * We will update this later once we have the actual interface names */ - networkHook = hooks.NewNetworkManagerHook(log) - networkHook.ExecuteWithUserScript(hooks.HookBeforeIfRead, config.HookScriptBeforeIfRead, params) + networkHook = hooks.NewNetworkManagerHook(log, params) + networkHook.ExecuteWithUserScript(hooks.HookBeforeIfRead, config.HookScriptBeforeIfRead) err = utils.ClearVppManagerFiles() if err != nil { @@ -183,20 +183,6 @@ func main() { log.Errorf("Error raising memlock limit, VPP may fail to start: %v", err) } - confs, err := startup.GetInterfaceConfig(params) - if err != nil { - log.Fatalf("Error getting initial interface configuration: %s", err) - } - - /* Update native Go NewNetworkManagerHook with all interface names */ - if len(params.UplinksSpecs) > 0 { - interfaceNames := make([]string, len(params.UplinksSpecs)) - for i, spec := range params.UplinksSpecs { - interfaceNames[i] = spec.InterfaceName - } - networkHook.SetInterfaceNames(interfaceNames) - } - runningCond = sync.NewCond(&sync.Mutex{}) go handleSignals() @@ -204,23 +190,24 @@ func main() { defer cancel() config.HandleUsr2Signal(ctx, log.WithFields(logrus.Fields{"component": "sighdlr"})) - startup.PrintVppManagerConfig(params, confs) - - runner := NewVPPRunner(params, confs) + startup.PrintVppManagerConfig(params) makeNewVPPIndex() - if len(params.UplinksSpecs) == 1 && params.UplinksSpecs[0].VppDriver == "" { - for _, driver := range uplink.SupportedUplinkDrivers(params, confs[0], ¶ms.UplinksSpecs[0]) { + if len(params.Interfaces) == 1 && params.InterfacesById[0].Spec.VppDriver == "" { + intf := params.InterfacesById[0] + for _, driver := range uplink.SupportedUplinkDrivers(params, intf) { err := utils.CleanupCoreFiles(config.GetCalicoVppInitialConfig().CorePattern, maxCoreFiles) if err != nil { log.Errorf("CleanupCoreFiles errored %s", err) } + intf.Driver = driver internalKill = false - err = runner.Run([]uplink.UplinkDriver{driver}) + runner := NewVPPRunner(params) + err = runner.Run() if err != nil { - networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored, params) + networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored) log.Errorf("VPP(%s) run failed with %s", driver.GetName(), err) } if vppProcess != nil && !internalKill { @@ -236,15 +223,10 @@ func main() { log.Errorf("CleanupCoreFiles errored %s", err) } - var drivers []uplink.UplinkDriver - for idx := 0; idx < len(params.UplinksSpecs); idx++ { - drivers = append(drivers, uplink.NewUplinkDriver(params.UplinksSpecs[idx].VppDriver, - params, confs[idx], ¶ms.UplinksSpecs[idx])) - } - - err = runner.Run(drivers) + runner := NewVPPRunner(params) + err = runner.Run() if err != nil { - networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored, params) + networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored) log.Errorf("VPP run failed with %v", err) } diff --git a/vpp-manager/startup/interface_config.go b/vpp-manager/startup/interface_config.go index b5c857ce..251c3523 100644 --- a/vpp-manager/startup/interface_config.go +++ b/vpp-manager/startup/interface_config.go @@ -16,10 +16,7 @@ package startup import ( - "encoding/gob" - "fmt" "net" - "os" "github.com/pkg/errors" log "github.com/sirupsen/logrus" @@ -30,56 +27,6 @@ import ( "github.com/projectcalico/vpp-dataplane/v3/vpplink" ) -func GetInterfaceConfig(params *config.VppManagerParams) (conf []*config.LinuxInterfaceState, err error) { - errs := []error{} - conf = []*config.LinuxInterfaceState{} - for _, ifSpec := range params.UplinksSpecs { - configuration, err := loadInterfaceConfigFromLinux(ifSpec) - errs = append(errs, err) - conf = append(conf, configuration) - } - allLoaded := true - for i := range errs { - if errs[i] != nil { - allLoaded = false - log.Warnf("Could not load config from linux (%v)", errs[i]) - } - } - if allLoaded { - err = saveConfig(params, conf) - if err != nil { - log.Warnf("Could not save interface config: %v", err) - } - } else { - // Loading config failed, try loading from save file - confFile, err2 := loadInterfaceConfigFromFile(params) - if err2 != nil { - return nil, err2 - } - // If loaded from file replace non loaded interface configs - for i := range conf { - if conf[i] == nil { - for j := range confFile { - if confFile[j].InterfaceName == params.UplinksSpecs[i].InterfaceName { - conf[i] = confFile[j] - } - } - } - } - for i := range conf { - if conf[i] == nil { - return nil, fmt.Errorf("interface configs not found") - } - } - log.Infof("Loaded config. Interfaces marked as down since loading config from linux failed.") - // This ensures we don't try to set the interface down in runVpp() - for _, config := range conf { - config.IsUp = false - } - } - return conf, nil -} - func loadInterfaceConfigFromLinux(ifSpec config.UplinkInterfaceSpec) (*config.LinuxInterfaceState, error) { conf := config.LinuxInterfaceState{} link, err := netlink.LinkByName(ifSpec.InterfaceName) @@ -146,53 +93,3 @@ func getNodeAddress(conf *config.LinuxInterfaceState, isV6 bool) string { } return "" } - -func clearSavedConfig(params *config.VppManagerParams) { - if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" { - return - } - err := os.Remove(config.GetCalicoVppInitialConfig().IfConfigSavePath) - if err != nil { - log.Warnf("could not delete saved interface config: %v", err) - } -} - -func saveConfig(params *config.VppManagerParams, conf []*config.LinuxInterfaceState) error { - if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" { - return nil - } - file, err := os.Create(config.GetCalicoVppInitialConfig().IfConfigSavePath) - if err != nil { - return errors.Wrap(err, "error opening save file") - } - enc := gob.NewEncoder(file) - err = enc.Encode(conf) - if err != nil { - file.Close() - clearSavedConfig(params) - return errors.Wrap(err, "error encoding data") - } - err = file.Close() - if err != nil { - return errors.Wrap(err, "error closing file") - } - return nil -} - -func loadInterfaceConfigFromFile(params *config.VppManagerParams) ([]*config.LinuxInterfaceState, error) { - conf := []*config.LinuxInterfaceState{} - if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" { - return nil, fmt.Errorf("interface config save file not configured") - } - file, err := os.Open(config.GetCalicoVppInitialConfig().IfConfigSavePath) - if err != nil { - return nil, errors.Wrap(err, "error opening save file") - } - dec := gob.NewDecoder(file) - err = dec.Decode(&conf) - if err != nil { - return nil, errors.Wrap(err, "decode error") - } - file.Close() - return conf, nil -} diff --git a/vpp-manager/startup/startup.go b/vpp-manager/startup/startup.go index ca4c8c92..adb2248a 100644 --- a/vpp-manager/startup/startup.go +++ b/vpp-manager/startup/startup.go @@ -21,51 +21,60 @@ import ( log "github.com/sirupsen/logrus" "github.com/projectcalico/vpp-dataplane/v3/config" + "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/uplink" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils" ) func NewVppManagerParams() *config.VppManagerParams { params := &config.VppManagerParams{ NodeAnnotations: utils.FetchNodeAnnotations(*config.NodeName), + Interfaces: make(map[string]*config.VppManagerInterface), + InterfacesById: make([]*config.VppManagerInterface, 0), } - /* uplink configuration: This is being deprecated */ + // uplink configuration: This is being deprecated if mainInterface := *config.InterfaceVar; mainInterface != "" { log.Warn("Use of CALICOVPP_INTERFACE, CALICOVPP_NATIVE_DRIVER and CALICOVPP_SWAP_DRIVER is deprecated, please use CALICOVPP_INTERFACES instead") - params.UplinksSpecs = []config.UplinkInterfaceSpec{{ - InterfaceName: mainInterface, - VppDriver: strings.ToLower(*config.NativeDriver), - NewDriverName: *config.SwapDriver, - }} + intf := &config.VppManagerInterface{ + Spec: config.UplinkInterfaceSpec{ + InterfaceName: mainInterface, + VppDriver: strings.ToLower(*config.NativeDriver), + NewDriverName: *config.SwapDriver, + }, + } + params.Interfaces[intf.Spec.InterfaceName] = intf + params.InterfacesById = append(params.InterfacesById, intf) } - /* uplinks configuration */ + // uplinks configuration isMainCount := 0 - for _, uplink := range config.GetCalicoVppInterfaces().UplinkInterfaces { - params.UplinksSpecs = append(params.UplinksSpecs, uplink) - if uplink.IsMain { + for _, spec := range config.GetCalicoVppInterfaces().UplinkInterfaces { + intf := &config.VppManagerInterface{Spec: spec} + params.Interfaces[intf.Spec.InterfaceName] = intf + params.InterfacesById = append(params.InterfacesById, intf) + if intf.Spec.IsMain { isMainCount++ } } - if len(params.UplinksSpecs) == 0 { + if len(params.Interfaces) == 0 { log.Panicf("No interface specified. Specify an interface through the environment variable") } if isMainCount == 0 { // By default the first interface is main - params.UplinksSpecs[0].IsMain = true + params.InterfacesById[0].Spec.IsMain = true } else if isMainCount > 1 { log.Panicf("Too many interfaces tagged Main") } - for index, uplink := range params.UplinksSpecs { - uplink.SetUplinkInterfaceIndex(index) - err := uplink.Validate(nil) + for index, intf := range params.InterfacesById { + intf.Spec.SetUplinkInterfaceIndex(index) + err := intf.Spec.Validate(nil) if err != nil { - log.Panicf("error validating uplink %s %s", uplink.String(), err) + log.Panicf("error validating uplink %s %s", intf.Spec.String(), err) } } - /* Drivers */ + // Drivers params.LoadedDrivers = make(map[string]bool) vfioLoaded, err := utils.IsDriverLoaded(config.DriverVfioPci) if err != nil { @@ -78,7 +87,7 @@ func NewVppManagerParams() *config.VppManagerParams { } params.LoadedDrivers[config.DriverUioPciGeneric] = uioLoaded - /* AF XDP support */ + // AF XDP support kernel, err := utils.GetOsKernelVersion() if err != nil { log.Warnf("Error getting os kernel version %v", err) @@ -86,7 +95,7 @@ func NewVppManagerParams() *config.VppManagerParams { params.KernelVersion = kernel } - /* Hugepages */ + // Hugepages nrHugepages, err := utils.GetNrHugepages() if err != nil { log.Warnf("Error getting nrHugepages %v", err) @@ -99,40 +108,50 @@ func NewVppManagerParams() *config.VppManagerParams { log.Warnf("Error getting vfio iommu state %v", err) } + for _, intf := range params.Interfaces { + uplinkState, err := loadInterfaceConfigFromLinux(intf.Spec) + if err != nil { + log.Panicf("Could not load config from linux (%v)", err) + } + intf.State = uplinkState + intf.Driver = uplink.NewUplinkDriver( + intf.Driver.GetName(), + params, + intf, + ) + } return params } -func PrintVppManagerConfig(params *config.VppManagerParams, confs []*config.LinuxInterfaceState) { +func PrintVppManagerConfig(params *config.VppManagerParams) { log.Infof("-- Environment --") log.Infof("Hugepages %d", params.AvailableHugePages) log.Infof("KernelVersion %s", params.KernelVersion) log.Infof("Drivers %v", params.LoadedDrivers) log.Infof("initial iommu status %s", params.InitialVfioEnableUnsafeNoIommuMode) - for _, ifSpec := range params.UplinksSpecs { + for _, intf := range params.Interfaces { log.Infof("-- Interface Spec --") - log.Infof("Interface Name: %s", ifSpec.InterfaceName) - log.Infof("Native Driver: %s", ifSpec.VppDriver) - log.Infof("New Drive Name: %s", ifSpec.NewDriverName) - log.Infof("PHY target #Queues rx:%d tx:%d", ifSpec.NumRxQueues, ifSpec.NumTxQueues) - log.Infof("Tap MTU: %d", ifSpec.Mtu) + log.Infof("Interface Name: %s", intf.Spec.InterfaceName) + log.Infof("Native Driver: %s", intf.Spec.VppDriver) + log.Infof("New Drive Name: %s", intf.Spec.NewDriverName) + log.Infof("PHY target #Queues rx:%d tx:%d", intf.Spec.NumRxQueues, intf.Spec.NumTxQueues) + log.Infof("Tap MTU: %d", intf.Spec.Mtu) - } - for _, conf := range confs { log.Infof("-- Interface config --") - log.Infof("Node IP4: %s", conf.NodeIP4) - log.Infof("Node IP6: %s", conf.NodeIP6) - log.Infof("PciID: %s", conf.PciID) - log.Infof("Driver: %s", conf.Driver) - log.Infof("Linux IF was up ? %t", conf.IsUp) - log.Infof("Promisc was on ? %t", conf.PromiscOn) - log.Infof("DoSwapDriver: %t", conf.DoSwapDriver) - log.Infof("Mac: %s", conf.HardwareAddr.String()) - log.Infof("Addresses: [%s]", conf.AddressString()) - log.Infof("Routes: [%s]", conf.RouteString()) - log.Infof("PHY original #Queues rx:%d tx:%d", conf.NumRxQueues, conf.NumTxQueues) - log.Infof("MTU %d", conf.Mtu) - log.Infof("isTunTap %t", conf.IsTunTap) - log.Infof("isVeth %t", conf.IsVeth) + log.Infof("Node IP4: %s", intf.State.NodeIP4) + log.Infof("Node IP6: %s", intf.State.NodeIP6) + log.Infof("PciID: %s", intf.State.PciID) + log.Infof("Driver: %s", intf.State.Driver) + log.Infof("Linux IF was up ? %t", intf.State.IsUp) + log.Infof("Promisc was on ? %t", intf.State.PromiscOn) + log.Infof("DoSwapDriver: %t", intf.State.DoSwapDriver) + log.Infof("Mac: %s", intf.State.HardwareAddr.String()) + log.Infof("Addresses: [%s]", intf.State.AddressString()) + log.Infof("Routes: [%s]", intf.State.RouteString()) + log.Infof("PHY original #Queues rx:%d tx:%d", intf.State.NumRxQueues, intf.State.NumTxQueues) + log.Infof("MTU %d", intf.State.Mtu) + log.Infof("isTunTap %t", intf.State.IsTunTap) + log.Infof("isVeth %t", intf.State.IsVeth) } } diff --git a/vpp-manager/uplink/af_packet.go b/vpp-manager/uplink/af_packet.go index 1289e356..5300e528 100644 --- a/vpp-manager/uplink/af_packet.go +++ b/vpp-manager/uplink/af_packet.go @@ -41,20 +41,20 @@ func (d *AFPacketDriver) IsSupported(warn bool) bool { func (d *AFPacketDriver) GetDefaultRxMode() types.RxMode { return types.InterruptRxMode } func (d *AFPacketDriver) PreconfigureLinux() error { - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.intf.Spec.InterfaceName) if err != nil { - return errors.Wrapf(err, "Error finding link %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Error finding link %s", d.intf.Spec.InterfaceName) } err = netlink.SetPromiscOn(link) if err != nil { - return errors.Wrapf(err, "Error set link %s promisc on", d.spec.InterfaceName) + return errors.Wrapf(err, "Error set link %s promisc on", d.intf.Spec.InterfaceName) } return nil } -func (d *AFPacketDriver) RestoreLinux(allInterfacesPhysical bool) { - if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) +func (d *AFPacketDriver) RestoreLinux() { + if !d.params.AllInterfacesPhysical() { + err := d.moveInterfaceFromNS(d.intf.Spec.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } @@ -64,9 +64,9 @@ func (d *AFPacketDriver) RestoreLinux(allInterfacesPhysical bool) { return } // Interface should pop back in root ns once vpp exits - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -74,7 +74,7 @@ func (d *AFPacketDriver) RestoreLinux(allInterfacesPhysical bool) { log.Infof("Setting promisc off") err = netlink.SetPromiscOff(link) if err != nil { - log.Errorf("Error setting link %s promisc off %v", d.spec.InterfaceName, err) + log.Errorf("Error setting link %s promisc off %v", d.intf.Spec.InterfaceName, err) } } @@ -96,7 +96,7 @@ func (d *AFPacketDriver) fetchBooleanAnnotation(annotation string, defaultValue } func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.intf.Spec.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } @@ -117,19 +117,20 @@ func (d *AFPacketDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int } log.Infof("Created AF_PACKET interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.intf.Spec.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.intf.Spec.InterfaceName) if err != nil { return err } return nil } -func NewAFPacketDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *AFPacketDriver { - d := &AFPacketDriver{} - d.name = NativeDriverAfPacket - d.conf = conf - d.params = params - d.spec = spec - return d +func NewAFPacketDriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *AFPacketDriver { + return &AFPacketDriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverAfPacket, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/af_xdp.go b/vpp-manager/uplink/af_xdp.go index 4b9d5ab0..2274e221 100644 --- a/vpp-manager/uplink/af_xdp.go +++ b/vpp-manager/uplink/af_xdp.go @@ -62,19 +62,19 @@ func (d *AFXDPDriver) IsSupported(warn bool) bool { } func (d *AFXDPDriver) PreconfigureLinux() error { - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.intf.Spec.InterfaceName) if err != nil { - return errors.Wrapf(err, "Error finding link %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Error finding link %s", d.intf.Spec.InterfaceName) } err = netlink.SetPromiscOn(link) if err != nil { - return errors.Wrapf(err, "Error setting link %s promisc on", d.spec.InterfaceName) + return errors.Wrapf(err, "Error setting link %s promisc on", d.intf.Spec.InterfaceName) } - err = utils.SetInterfaceRxQueues(d.spec.InterfaceName, d.spec.NumRxQueues) + err = utils.SetInterfaceRxQueues(d.intf.Spec.InterfaceName, d.intf.Spec.NumRxQueues) if err != nil { - log.Errorf("Error setting link %s NumQueues to %d, using %d queues: %v", d.spec.InterfaceName, d.spec.NumRxQueues, d.conf.NumRxQueues, err) + log.Errorf("Error setting link %s NumQueues to %d, using %d queues: %v", d.intf.Spec.InterfaceName, d.intf.Spec.NumRxQueues, d.conf.NumRxQueues, err) /* Try with linux NumRxQueues on error, otherwise af_xdp wont start */ - d.spec.NumRxQueues = d.conf.NumRxQueues + d.intf.Spec.NumRxQueues = d.conf.NumRxQueues } if d.conf.Mtu > maxAfXDPMTU { log.Infof("Reducing interface MTU to %d for AF_XDP", maxAfXDPMTU) @@ -84,16 +84,16 @@ func (d *AFXDPDriver) PreconfigureLinux() error { } d.conf.Mtu = maxAfXDPMTU } - if d.spec.Mtu > maxAfXDPMTU { + if d.intf.Spec.Mtu > maxAfXDPMTU { log.Infof("Reducing user specified MTU to %d", maxAfXDPMTU) - d.spec.Mtu = maxAfXDPMTU + d.intf.Spec.Mtu = maxAfXDPMTU } return nil } -func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { - if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) +func (d *AFXDPDriver) RestoreLinux() { + if !d.params.AllInterfacesPhysical() { + err := d.moveInterfaceFromNS(d.intf.Spec.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } @@ -103,9 +103,9 @@ func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { return } // Interface should pop back in root ns once vpp exits - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -115,14 +115,14 @@ func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { log.Infof("Setting promisc off") err = netlink.SetPromiscOff(link) if err != nil { - log.Errorf("Error setting link %s promisc off %v", d.spec.InterfaceName, err) + log.Errorf("Error setting link %s promisc off %v", d.intf.Spec.InterfaceName, err) } } - if d.conf.NumRxQueues != d.spec.NumRxQueues { + if d.conf.NumRxQueues != d.intf.Spec.NumRxQueues { log.Infof("Setting back %d queues", d.conf.NumRxQueues) - err = utils.SetInterfaceRxQueues(d.spec.InterfaceName, d.conf.NumRxQueues) + err = utils.SetInterfaceRxQueues(d.intf.Spec.InterfaceName, d.conf.NumRxQueues) if err != nil { - log.Errorf("Error setting link %s NumQueues to %d %v", d.spec.InterfaceName, d.conf.NumRxQueues, err) + log.Errorf("Error setting link %s NumQueues to %d %v", d.intf.Spec.InterfaceName, d.conf.NumRxQueues, err) } } @@ -131,7 +131,7 @@ func (d *AFXDPDriver) RestoreLinux(allInterfacesPhysical bool) { } func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.intf.Spec.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } @@ -149,19 +149,20 @@ func (d *AFXDPDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, u if err != nil { return errors.Wrap(err, "could not set af_xdp interface mac address in vpp") } - d.spec.SwIfIndex = intf.SwIfIndex - err = d.TagMainInterface(vpp, intf.SwIfIndex, d.spec.InterfaceName) + d.intf.Spec.SwIfIndex = intf.SwIfIndex + err = d.TagMainInterface(vpp, intf.SwIfIndex, d.intf.Spec.InterfaceName) if err != nil { return err } return nil } -func NewAFXDPDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *AFXDPDriver { - d := &AFXDPDriver{} - d.name = NativeDriverAfXdp - d.conf = conf - d.params = params - d.spec = spec - return d +func NewAFXDPDriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *AFXDPDriver { + return &AFXDPDriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverAfXdp, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/avf.go b/vpp-manager/uplink/avf.go index d50a2acf..43e6ac11 100644 --- a/vpp-manager/uplink/avf.go +++ b/vpp-manager/uplink/avf.go @@ -52,9 +52,9 @@ func (d *AVFDriver) IsSupported(warn bool) (supported bool) { } func (d *AVFDriver) PreconfigureLinux() (err error) { - pciID, err := utils.GetInterfacePciID(d.spec.InterfaceName) + pciID, err := utils.GetInterfacePciID(d.intf.Spec.InterfaceName) if err != nil { - return errors.Wrapf(err, "cannot get interface %s pciID", d.spec.InterfaceName) + return errors.Wrapf(err, "cannot get interface %s pciID", d.intf.Spec.InterfaceName) } numVFs, err := utils.GetInterfaceNumVFs(pciID) @@ -66,34 +66,34 @@ func (d *AVFDriver) PreconfigureLinux() (err error) { /* This is a PF */ d.pfPCI = pciID if numVFs == 0 { - log.Infof("Creating a VF for %s", d.spec.InterfaceName) + log.Infof("Creating a VF for %s", d.intf.Spec.InterfaceName) err := utils.CreateInterfaceVF(pciID) if err != nil { - return errors.Wrapf(err, "Couldnt create VF for %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt create VF for %s", d.intf.Spec.InterfaceName) } /* Create a mac for the new VF */ - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.intf.Spec.InterfaceName) if err != nil { - return errors.Wrapf(err, "Couldnt find Interface %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt find Interface %s", d.intf.Spec.InterfaceName) } hardwareAddr := utils.CycleHardwareAddr(d.conf.HardwareAddr, 7) err = netlink.LinkSetVfHardwareAddr(link, 0 /* vf */, hardwareAddr) if err != nil { - return errors.Wrapf(err, "Couldnt set VF 0 hwaddr %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt set VF 0 hwaddr %s", d.intf.Spec.InterfaceName) } } vfPCI, err := utils.GetInterfaceVFPciID(pciID) if err != nil { - return errors.Wrapf(err, "Couldnt get VF pciID for %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt get VF pciID for %s", d.intf.Spec.InterfaceName) } d.vfPCI = vfPCI } if d.pfPCI != "" { - err := utils.SetVFSpoofTrust(d.spec.InterfaceName, 0 /* vf */, false /* spoof */, true /* trust */) + err := utils.SetVFSpoofTrust(d.intf.Spec.InterfaceName, 0 /* vf */, false /* spoof */, true /* trust */) if err != nil { - return errors.Wrapf(err, "Couldnt set VF spoof off trust on %s", d.spec.InterfaceName) + return errors.Wrapf(err, "Couldnt set VF spoof off trust on %s", d.intf.Spec.InterfaceName) } } @@ -113,10 +113,10 @@ func (d *AVFDriver) PreconfigureLinux() (err error) { return nil } -func (d *AVFDriver) RestoreLinux(allInterfacesPhysical bool) { - if !allInterfacesPhysical { +func (d *AVFDriver) RestoreLinux() { + if !d.params.AllInterfacesPhysical() { if d.pfPCI != "" { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + err := d.moveInterfaceFromNS(d.intf.Spec.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } @@ -127,9 +127,9 @@ func (d *AVFDriver) RestoreLinux(allInterfacesPhysical bool) { } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -141,7 +141,7 @@ func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, upl if d.pfPCI != "" { /* We were passed a PF, move it to vpp's NS so it doesn't conflict with vpptap0 */ - err := d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err := d.moveInterfaceToNS(d.intf.Spec.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } @@ -162,19 +162,20 @@ func (d *AVFDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, upl return errors.Wrapf(err, "Error setting AVF promisc on") } - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.intf.Spec.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.intf.Spec.InterfaceName) if err != nil { return err } return nil } -func NewAVFDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *AVFDriver { - d := &AVFDriver{} - d.name = NativeDriverAvf - d.conf = conf - d.params = params - d.spec = spec - return d +func NewAVFDriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *AVFDriver { + return &AVFDriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverAvf, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/common.go b/vpp-manager/uplink/common.go index a86313a5..e6e6b4a0 100644 --- a/vpp-manager/uplink/common.go +++ b/vpp-manager/uplink/common.go @@ -45,17 +45,7 @@ type UplinkDriverData struct { conf *config.LinuxInterfaceState params *config.VppManagerParams name string - spec *config.UplinkInterfaceSpec -} - -type UplinkDriver interface { - PreconfigureLinux() error - CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) error - RestoreLinux(allInterfacesPhysical bool) - IsSupported(warn bool) bool - GetName() string - UpdateVppConfigFile(template string) string - GetDefaultRxMode() types.RxMode + intf *config.VppManagerInterface } func (d *UplinkDriverData) GetDefaultRxMode() types.RxMode { return types.AdaptativeRxMode } @@ -113,9 +103,9 @@ func (d *UplinkDriverData) moveInterfaceToNS(ifName string, pid int) error { } func (d *UplinkDriverData) removeLinuxIfConf(setIfDown bool) { - link, err := netlink.LinkByName(d.spec.InterfaceName) + link, err := netlink.LinkByName(d.intf.Spec.InterfaceName) if err != nil { - log.Errorf("Error finding link %s: %s", d.spec.InterfaceName, err) + log.Errorf("Error finding link %s: %s", d.intf.Spec.InterfaceName, err) } else { // Remove routes to not have them conflict with vpptap0 for _, route := range d.conf.Routes { @@ -138,7 +128,7 @@ func (d *UplinkDriverData) removeLinuxIfConf(setIfDown bool) { if err != nil { // In case it still succeeded err2 := netlink.LinkSetUp(link) - log.Errorf("Error setting link %s down: %s (err2 %s)", d.spec.InterfaceName, err, err2) + log.Errorf("Error setting link %s down: %s (err2 %s)", d.intf.Spec.InterfaceName, err, err2) } } } @@ -177,60 +167,62 @@ func (d *UplinkDriverData) UpdateVppConfigFile(template string) string { func (d *UplinkDriverData) getGenericVppInterface() types.GenericVppInterface { return types.GenericVppInterface{ - NumRxQueues: d.spec.NumRxQueues, - RxQueueSize: d.spec.RxQueueSize, - TxQueueSize: d.spec.TxQueueSize, - NumTxQueues: d.spec.NumTxQueues, + NumRxQueues: d.intf.Spec.NumRxQueues, + RxQueueSize: d.intf.Spec.RxQueueSize, + TxQueueSize: d.intf.Spec.TxQueueSize, + NumTxQueues: d.intf.Spec.NumTxQueues, HardwareAddr: d.conf.HardwareAddr, - HostInterfaceName: d.spec.InterfaceName, + HostInterfaceName: d.intf.Spec.InterfaceName, } } -func SupportedUplinkDrivers(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) []UplinkDriver { - lst := make([]UplinkDriver, 0) +func SupportedUplinkDrivers(params *config.VppManagerParams, + intf *config.VppManagerInterface) []config.UplinkDriver { + lst := make([]config.UplinkDriver, 0) - if d := NewVirtioDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewVirtioDriver(params, intf); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewAVFDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewAVFDriver(params, intf); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewRDMADriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewRDMADriver(params, intf); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewVmxnet3Driver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewVmxnet3Driver(params, intf); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewAFXDPDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewAFXDPDriver(params, intf); d.IsSupported(false /* warn */) { lst = append(lst, d) } - if d := NewAFPacketDriver(params, conf, spec); d.IsSupported(false /* warn */) { + if d := NewAFPacketDriver(params, intf); d.IsSupported(false /* warn */) { lst = append(lst, d) } return lst } -func NewUplinkDriver(name string, params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) (d UplinkDriver) { +func NewUplinkDriver(name string, params *config.VppManagerParams, + intf *config.VppManagerInterface) (d config.UplinkDriver) { switch name { case NativeDriverRdma: - d = NewRDMADriver(params, conf, spec) + d = NewRDMADriver(params, intf) case NativeDriverVmxnet3: - d = NewVmxnet3Driver(params, conf, spec) + d = NewVmxnet3Driver(params, intf) case NativeDriverAfPacket: - d = NewAFPacketDriver(params, conf, spec) + d = NewAFPacketDriver(params, intf) case NativeDriverAfXdp: - d = NewAFXDPDriver(params, conf, spec) + d = NewAFXDPDriver(params, intf) case NativeDriverVirtio: - d = NewVirtioDriver(params, conf, spec) + d = NewVirtioDriver(params, intf) case NativeDriverAvf: - d = NewAVFDriver(params, conf, spec) + d = NewAVFDriver(params, intf) case NativeDriverDpdk: - d = NewDPDKDriver(params, conf, spec) + d = NewDPDKDriver(params, intf) case NativeDriverNone: fallthrough default: log.Warnf("Using default driver") - d = NewDefaultDriver(params, conf, spec) + d = NewDefaultDriver(params, intf) } d.IsSupported(true /* warn */) return d diff --git a/vpp-manager/uplink/default.go b/vpp-manager/uplink/default.go index 507254ae..0bba0f5e 100644 --- a/vpp-manager/uplink/default.go +++ b/vpp-manager/uplink/default.go @@ -46,16 +46,16 @@ func (d *DefaultDriver) PreconfigureLinux() (err error) { if d.conf.PciID == "" { log.Warnf("PCI ID not found, not swapping drivers") } else { - err = utils.SwapDriver(d.conf.PciID, d.spec.NewDriverName, true) + err = utils.SwapDriver(d.conf.PciID, d.intf.Spec.NewDriverName, true) if err != nil { - log.Warnf("Failed to swap driver to %s: %v", d.spec.NewDriverName, err) + log.Warnf("Failed to swap driver to %s: %v", d.intf.Spec.NewDriverName, err) } } } return nil } -func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) { +func (d *DefaultDriver) RestoreLinux() { if d.conf.PciID != "" && d.conf.Driver != "" { err := utils.SwapDriver(d.conf.PciID, d.conf.Driver, false) if err != nil { @@ -67,9 +67,9 @@ func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) { } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -79,29 +79,30 @@ func (d *DefaultDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *DefaultDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // If interface is still in the host, move it to vpp netns to allow creation of the tap - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.intf.Spec.InterfaceName, vppPid) if err != nil { - log.Infof("Did NOT move interface %s to VPP netns: %v", d.spec.InterfaceName, err) + log.Infof("Did NOT move interface %s to VPP netns: %v", d.intf.Spec.InterfaceName, err) } else { - log.Infof("Moved interface %s to VPP netns", d.spec.InterfaceName) + log.Infof("Moved interface %s to VPP netns", d.intf.Spec.InterfaceName) } // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.IsMain { + if !d.intf.Spec.IsMain { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } - swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) + swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.intf.Spec.InterfaceName) if err != nil { - return fmt.Errorf("error trying to find interface with tag main-%s", d.spec.InterfaceName) + return fmt.Errorf("error trying to find interface with tag main-%s", d.intf.Spec.InterfaceName) } - d.spec.SwIfIndex = swIfIndex + d.intf.Spec.SwIfIndex = swIfIndex return nil } -func NewDefaultDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *DefaultDriver { - d := &DefaultDriver{} - d.name = NativeDriverNone - d.conf = conf - d.params = params - d.spec = spec - return d +func NewDefaultDriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *DefaultDriver { + return &DefaultDriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverNone, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/dpdk.go b/vpp-manager/uplink/dpdk.go index 2b76fbd3..f62a8fe2 100644 --- a/vpp-manager/uplink/dpdk.go +++ b/vpp-manager/uplink/dpdk.go @@ -48,7 +48,7 @@ func (d *DPDKDriver) IsSupported(warn bool) (supported bool) { func (d *DPDKDriver) getFinalDriver() string { if d.conf.DoSwapDriver { - return d.spec.NewDriverName + return d.intf.Spec.NewDriverName } return d.conf.Driver } @@ -56,9 +56,9 @@ func (d *DPDKDriver) getFinalDriver() string { func (d *DPDKDriver) PreconfigureLinux() (err error) { d.removeLinuxIfConf(true /* down */) if d.conf.DoSwapDriver { - err = utils.SwapDriver(d.conf.PciID, d.spec.NewDriverName, true) + err = utils.SwapDriver(d.conf.PciID, d.intf.Spec.NewDriverName, true) if err != nil { - log.Warnf("Failed to swap driver to %s: %v", d.spec.NewDriverName, err) + log.Warnf("Failed to swap driver to %s: %v", d.intf.Spec.NewDriverName, err) } } if d.getFinalDriver() == config.DriverVfioPci && @@ -84,15 +84,15 @@ func (d *DPDKDriver) UpdateVppConfigFile(template string) string { if d.params.AvailableHugePages > 0 { template = fmt.Sprintf( "%s\ndpdk {\ndev %s { num-rx-queues %d num-tx-queues %d num-rx-desc %d num-tx-desc %d tag %s } \n}\n", - template, d.conf.PciID, d.spec.NumRxQueues, d.spec.NumTxQueues, - d.spec.RxQueueSize, d.spec.TxQueueSize, "main-"+d.spec.InterfaceName, + template, d.conf.PciID, d.intf.Spec.NumRxQueues, d.intf.Spec.NumTxQueues, + d.intf.Spec.RxQueueSize, d.intf.Spec.TxQueueSize, "main-"+d.intf.Spec.InterfaceName, ) } else { template = fmt.Sprintf( "%s\ndpdk {\niova-mode va\nno-hugetlb\ndev %s { num-rx-queues %d num-tx-queues %d num-rx-desc %d num-tx-desc %d tag %s } \n}\n", - template, d.conf.PciID, d.spec.NumRxQueues, d.spec.NumTxQueues, - d.spec.RxQueueSize, d.spec.TxQueueSize, "main-"+d.spec.InterfaceName, + template, d.conf.PciID, d.intf.Spec.NumRxQueues, d.intf.Spec.NumTxQueues, + d.intf.Spec.RxQueueSize, d.intf.Spec.TxQueueSize, "main-"+d.intf.Spec.InterfaceName, ) // If no hugepages, also edit `buffers {}` @@ -118,21 +118,21 @@ func (d *DPDKDriver) restoreInterfaceName() error { if err != nil { return errors.Wrapf(err, "Error getting new if name for %s: %v", newName, d.conf.PciID) } - if newName == d.spec.InterfaceName { + if newName == d.intf.Spec.InterfaceName { return nil } link, err := netlink.LinkByName(newName) if err != nil { return errors.Wrapf(err, "Error getting new link %s: %v", newName, link) } - err = netlink.LinkSetName(link, d.spec.InterfaceName) + err = netlink.LinkSetName(link, d.intf.Spec.InterfaceName) if err != nil { - return errors.Wrapf(err, "Error setting new if name for %s: %v", d.spec.InterfaceName, link) + return errors.Wrapf(err, "Error setting new if name for %s: %v", d.intf.Spec.InterfaceName, link) } return nil } -func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { +func (d *DPDKDriver) RestoreLinux() { if d.conf.PciID != "" && d.conf.Driver != "" { err := utils.SwapDriver(d.conf.PciID, d.conf.Driver, false) if err != nil { @@ -155,9 +155,9 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -177,15 +177,15 @@ func (d *DPDKDriver) RestoreLinux(allInterfacesPhysical bool) { func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *config.UplinkInterfaceSpec) (err error) { // Nothing to do VPP autocreates on startup // refusing to run on secondary interfaces as we have no way to figure out the sw_if_index - if !d.spec.IsMain { + if !d.intf.Spec.IsMain { return fmt.Errorf("%s driver not supported for secondary interfaces", d.name) } - swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.spec.InterfaceName) + swIfIndex, err := vpp.SearchInterfaceWithTag("main-" + d.intf.Spec.InterfaceName) if err != nil || swIfIndex == ^uint32(0) { - return fmt.Errorf("error trying to find interface with tag main-%s", d.spec.InterfaceName) + return fmt.Errorf("error trying to find interface with tag main-%s", d.intf.Spec.InterfaceName) } - log.Debugf("Found interface with swIfIndex %d for %s", swIfIndex, d.spec.InterfaceName) - d.spec.SwIfIndex = swIfIndex + log.Debugf("Found interface with swIfIndex %d for %s", swIfIndex, d.intf.Spec.InterfaceName) + d.intf.Spec.SwIfIndex = swIfIndex err = vpp.SetInterfaceMacAddress(swIfIndex, d.conf.HardwareAddr) if err != nil && gerrors.Is(err, types.VppErrorUnimplemented) { log.Warn("Setting dpdk interface mac address in vpp unsupported") @@ -195,11 +195,12 @@ func (d *DPDKDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, up return nil } -func NewDPDKDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *DPDKDriver { - d := &DPDKDriver{} - d.name = NativeDriverDpdk - d.conf = conf - d.params = params - d.spec = spec - return d +func NewDPDKDriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *DPDKDriver { + return &DPDKDriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverDpdk, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/rdma.go b/vpp-manager/uplink/rdma.go index 8862e835..0fd8a3b2 100644 --- a/vpp-manager/uplink/rdma.go +++ b/vpp-manager/uplink/rdma.go @@ -47,9 +47,9 @@ func (d *RDMADriver) PreconfigureLinux() (err error) { return nil } -func (d *RDMADriver) RestoreLinux(allInterfacesPhysical bool) { - if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) +func (d *RDMADriver) RestoreLinux() { + if !d.params.AllInterfacesPhysical() { + err := d.moveInterfaceFromNS(d.intf.Spec.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } @@ -59,9 +59,9 @@ func (d *RDMADriver) RestoreLinux(allInterfacesPhysical bool) { } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -79,26 +79,27 @@ func (d *RDMADriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, up return errors.Wrapf(err, "Error creating RDMA interface") } - err = d.moveInterfaceToNS(d.spec.InterfaceName, vppPid) + err = d.moveInterfaceToNS(d.intf.Spec.InterfaceName, vppPid) if err != nil { return errors.Wrap(err, "Moving uplink in NS failed") } log.Infof("Created RDMA interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.intf.Spec.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.intf.Spec.InterfaceName) if err != nil { return err } return nil } -func NewRDMADriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *RDMADriver { - d := &RDMADriver{} - d.name = NativeDriverRdma - d.conf = conf - d.params = params - d.spec = spec - return d +func NewRDMADriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *RDMADriver { + return &RDMADriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverRdma, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/virtio.go b/vpp-manager/uplink/virtio.go index d27d98d6..f0c2acac 100644 --- a/vpp-manager/uplink/virtio.go +++ b/vpp-manager/uplink/virtio.go @@ -55,7 +55,7 @@ func (d *VirtioDriver) IsSupported(warn bool) (supported bool) { } func (d *VirtioDriver) PreconfigureLinux() (err error) { - newDriverName := d.spec.NewDriverName + newDriverName := d.intf.Spec.NewDriverName doSwapDriver := d.conf.DoSwapDriver if newDriverName == "" { newDriverName = config.DriverVfioPci @@ -78,7 +78,7 @@ func (d *VirtioDriver) PreconfigureLinux() (err error) { return nil } -func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) { +func (d *VirtioDriver) RestoreLinux() { if d.params.InitialVfioEnableUnsafeNoIommuMode == config.VfioUnsafeNoIommuModeNO { err := utils.SetVfioEnableUnsafeNoIommuMode(config.VfioUnsafeNoIommuModeNO) if err != nil { @@ -91,8 +91,8 @@ func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) { log.Warnf("Error swapping back driver to %s for %s: %v", d.conf.Driver, d.conf.PciID, err) } } - if !allInterfacesPhysical { - err := d.moveInterfaceFromNS(d.spec.InterfaceName) + if !d.params.AllInterfacesPhysical() { + err := d.moveInterfaceFromNS(d.intf.Spec.InterfaceName) if err != nil { log.Warnf("Moving uplink back from NS failed %s", err) } @@ -102,9 +102,9 @@ func (d *VirtioDriver) RestoreLinux(allInterfacesPhysical bool) { } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -123,19 +123,20 @@ func (d *VirtioDriver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, } log.Infof("Created VIRTIO interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.intf.Spec.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.intf.Spec.InterfaceName) if err != nil { return err } return nil } -func NewVirtioDriver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *VirtioDriver { - d := &VirtioDriver{} - d.name = NativeDriverVirtio - d.conf = conf - d.params = params - d.spec = spec - return d +func NewVirtioDriver(params *config.VppManagerParams, intf *config.VppManagerInterface) *VirtioDriver { + return &VirtioDriver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverVirtio, + params: params, + intf: intf, + }, + } } diff --git a/vpp-manager/uplink/vmxnet3.go b/vpp-manager/uplink/vmxnet3.go index eeaea389..5566d0b9 100644 --- a/vpp-manager/uplink/vmxnet3.go +++ b/vpp-manager/uplink/vmxnet3.go @@ -63,7 +63,7 @@ func (d *Vmxnet3Driver) PreconfigureLinux() (err error) { return nil } -func (d *Vmxnet3Driver) RestoreLinux(allInterfacesPhysical bool) { +func (d *Vmxnet3Driver) RestoreLinux() { if d.conf.PciID != "" && d.conf.Driver != "" { err := utils.SwapDriver(d.conf.PciID, d.conf.Driver, true) if err != nil { @@ -76,9 +76,9 @@ func (d *Vmxnet3Driver) RestoreLinux(allInterfacesPhysical bool) { } // This assumes the link has kept the same name after the rebind. // It should be always true on systemd based distros - link, err := utils.SafeSetInterfaceUpByName(d.spec.InterfaceName) + link, err := utils.SafeSetInterfaceUpByName(d.intf.Spec.InterfaceName) if err != nil { - log.Warnf("Error setting %s up: %v", d.spec.InterfaceName, err) + log.Warnf("Error setting %s up: %v", d.intf.Spec.InterfaceName, err) return } @@ -106,19 +106,20 @@ func (d *Vmxnet3Driver) CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, log.Infof("Created Vmxnet3 interface %d", swIfIndex) - d.spec.SwIfIndex = swIfIndex - err = d.TagMainInterface(vpp, swIfIndex, d.spec.InterfaceName) + d.intf.Spec.SwIfIndex = swIfIndex + err = d.TagMainInterface(vpp, swIfIndex, d.intf.Spec.InterfaceName) if err != nil { return err } return nil } -func NewVmxnet3Driver(params *config.VppManagerParams, conf *config.LinuxInterfaceState, spec *config.UplinkInterfaceSpec) *Vmxnet3Driver { - d := &Vmxnet3Driver{} - d.name = NativeDriverVmxnet3 - d.conf = conf - d.params = params - d.spec = spec - return d +func NewVmxnet3Driver(params *config.VppManagerParams, intf *config.VppManagerInterface) *Vmxnet3Driver { + return &Vmxnet3Driver{ + UplinkDriverData: UplinkDriverData{ + name: NativeDriverVmxnet3, + intf: intf, + params: params, + }, + } } diff --git a/vpp-manager/utils/utils.go b/vpp-manager/utils/utils.go index aaca5c29..a8e3c3dc 100644 --- a/vpp-manager/utils/utils.go +++ b/vpp-manager/utils/utils.go @@ -783,3 +783,24 @@ func PrintLastBackTrace(coreFile string) { } } } + +func PingCalicoVpp() error { + dat, err := os.ReadFile(config.CalicoVppPidFile) + if err != nil { + if errors.Is(err, os.ErrNotExist) { + log.Infof("calico-vpp-pid file doesn't exist. Agent probably not started") + return nil + } + return errors.Wrapf(err, "Error reading %s", config.CalicoVppPidFile) + } + pid, err := strconv.ParseInt(strings.TrimSpace(string(dat[:])), 10, 32) + if err != nil { + return errors.Wrapf(err, "Error parsing %s", dat) + } + err = syscall.Kill(int(pid), syscall.SIGUSR1) + if err != nil { + return errors.Wrapf(err, "Error kill -SIGUSR1 %d", int(pid)) + } + log.Infof("Did kill -SIGUSR1 %d", int(pid)) + return nil +} diff --git a/vpp-manager/vpp_runner.go b/vpp-manager/vpp_runner.go index d361ff86..9417f434 100644 --- a/vpp-manager/vpp_runner.go +++ b/vpp-manager/vpp_runner.go @@ -21,8 +21,6 @@ import ( "net" "os" "os/exec" - "strconv" - "strings" "syscall" "time" @@ -32,140 +30,67 @@ import ( calicov3cli "github.com/projectcalico/calico/libcalico-go/lib/clientv3" calicoopts "github.com/projectcalico/calico/libcalico-go/lib/options" "github.com/vishvananda/netlink" - "gopkg.in/tomb.v2" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/cni/podinterface" "github.com/projectcalico/vpp-dataplane/v3/calico-vpp-agent/common" "github.com/projectcalico/vpp-dataplane/v3/config" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/hooks" - "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/uplink" "github.com/projectcalico/vpp-dataplane/v3/vpp-manager/utils" "github.com/projectcalico/vpp-dataplane/v3/vpplink" "github.com/projectcalico/vpp-dataplane/v3/vpplink/types" ) type VppRunner struct { - params *config.VppManagerParams - conf []*config.LinuxInterfaceState - vpp *vpplink.VppLink - uplinkDriver []uplink.UplinkDriver + params *config.VppManagerParams + vpp *vpplink.VppLink } -func NewVPPRunner(params *config.VppManagerParams, confs []*config.LinuxInterfaceState) *VppRunner { +func NewVPPRunner(params *config.VppManagerParams) *VppRunner { return &VppRunner{ params: params, - conf: confs, } } -func (v *VppRunner) GenerateVppConfigExecFile() error { - template, err := config.TemplateScriptReplace(*config.ConfigExecTemplate, v.params, v.conf) - if err != nil { - return err +func (v *VppRunner) Run() error { + for interfaceName, intf := range v.params.Interfaces { + log.Infof("Running interface %s with uplink %s", interfaceName, intf.Driver.GetName()) } - err = errors.Wrapf( - os.WriteFile(config.VppConfigExecFile, []byte(template+"\n"), 0744), - "Error writing VPP Exec configuration to %s", - config.VppConfigExecFile, - ) - return err -} - -func (v *VppRunner) GenerateVppConfigFile(drivers []uplink.UplinkDriver) error { - template, err := config.TemplateScriptReplace(*config.ConfigTemplate, v.params, v.conf) + template, err := config.TemplateScriptReplace(*config.ConfigExecTemplate, v.params) if err != nil { - return err + return errors.Wrap(err, "Error generating VPP config exec file") } - for _, driver := range drivers { - template = driver.UpdateVppConfigFile(template) + err = os.WriteFile(config.VppConfigExecFile, []byte(template+"\n"), 0744) + if err != nil { + return errors.Wrapf(err, "Error writing VPP config exec file to %s", config.VppConfigExecFile) } - err = errors.Wrapf( - os.WriteFile(config.VppConfigFile, []byte(template+"\n"), 0644), - "Error writing VPP configuration to %s", - config.VppConfigFile, - ) - return err -} -func (v *VppRunner) Run(drivers []uplink.UplinkDriver) error { - v.uplinkDriver = drivers - for idx := range v.conf { - log.Infof("Running with uplink %s", drivers[idx].GetName()) - } - err := v.GenerateVppConfigExecFile() + template, err = config.TemplateScriptReplace(*config.ConfigTemplate, v.params) if err != nil { - return errors.Wrap(err, "Error generating VPP config Exec") + return errors.Wrap(err, "Error generating VPP config file") } - - err = v.GenerateVppConfigFile(drivers) + for _, intf := range v.params.Interfaces { + template = intf.Driver.UpdateVppConfigFile(template) + } + err = os.WriteFile(config.VppConfigFile, []byte(template+"\n"), 0644) if err != nil { - return errors.Wrap(err, "Error generating VPP config") + return errors.Wrapf(err, "Error writing VPP config file to %s", config.VppConfigFile) } - for idx := range v.conf { - err = v.uplinkDriver[idx].PreconfigureLinux() + for _, intf := range v.params.Interfaces { + err = intf.Driver.PreconfigureLinux() if err != nil { - return errors.Wrapf(err, "Error pre-configuring Linux main IF: %s", v.uplinkDriver[idx]) + return errors.Wrapf(err, "Error pre-configuring Linux main IF: %s", intf.Driver.GetName()) } } - networkHook.ExecuteWithUserScript(hooks.HookBeforeVppRun, config.HookScriptBeforeVppRun, v.params) + networkHook.ExecuteWithUserScript(hooks.HookBeforeVppRun, config.HookScriptBeforeVppRun) err = v.runVpp() if err != nil { return errors.Wrapf(err, "Error running VPP") } - networkHook.ExecuteWithUserScript(hooks.HookVppDoneOk, config.HookScriptVppDoneOk, v.params) - return nil -} - -func (v *VppRunner) configureGlobalPunt() (err error) { - for _, ipFamily := range vpplink.IPFamilies { - err = v.vpp.PuntRedirect(types.IPPuntRedirect{ - RxSwIfIndex: vpplink.InvalidID, - Paths: []types.RoutePath{{ - Table: common.PuntTableID, - SwIfIndex: types.InvalidID, - }}, - }, ipFamily.IsIP6) - if err != nil { - return errors.Wrapf(err, "Error configuring punt redirect") - } - - err = v.vpp.PuntAllL4(ipFamily.IsIP6) - if err != nil { - return errors.Wrapf(err, "Error configuring L4 punt") - } - } - return -} - -func (v *VppRunner) configurePunt(tapSwIfIndex uint32, ifState config.LinuxInterfaceState) (err error) { - err = v.vpp.AddNeighbor(&types.Neighbor{ - SwIfIndex: tapSwIfIndex, - IP: config.VppHostPuntFakeGatewayAddress, - HardwareAddr: ifState.HardwareAddr, - Flags: types.IPNeighborStatic, - }) - if err != nil { - return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress) - } - /* In the punt table (where all punted traffics ends), route to the tap */ - for _, address := range ifState.Addresses { - err = v.vpp.RouteAdd(&types.Route{ - Dst: address.IPNet, - Table: common.PuntTableID, - Paths: []types.RoutePath{{ - Gw: config.VppHostPuntFakeGatewayAddress, - SwIfIndex: tapSwIfIndex, - }}, - }) - if err != nil { - return errors.Wrapf(err, "error adding vpp side routes for interface") - } - } - + networkHook.ExecuteWithUserScript(hooks.HookVppDoneOk, config.HookScriptVppDoneOk) return nil } @@ -299,262 +224,136 @@ func (v *VppRunner) configureLinuxTap(link netlink.Link, ifState config.LinuxInt return fakeNextHopIP4, fakeNextHopIP6, nil } -func (v *VppRunner) addExtraAddresses(addrList []netlink.Addr, extraAddrCount int, vppIfSwIfIndex uint32) (err error) { - ipFlowHash := types.FlowHashSrcIP | - types.FlowHashDstIP | - types.FlowHashSrcPort | - types.FlowHashDstPort | - types.FlowHashSymetric - - err = v.vpp.SetIPFlowHash(ipFlowHash, 0 /* vrf */, false /* isIPv6 */) - if err != nil { - log.Errorf("cannot configure flow hash: %v", err) - } - /* No v6 as extraAddrCount doesnt support it & flow hash breaks in vpp */ - - log.Infof("Adding %d extra addresses", extraAddrCount) - v4Count := 0 - var addr net.IPNet - for _, a := range addrList { - if a.IP.To4() != nil { - v4Count++ - addr = *a.IPNet - } - } - if v4Count != 1 { - return fmt.Errorf("%d IPv4 addresses found, not configuring extra addresses (need exactly 1)", v4Count) - } - for i := 1; i <= extraAddrCount; i++ { - a := &net.IPNet{ - IP: net.IP(append([]byte(nil), addr.IP.To4()...)), - Mask: addr.Mask, - } - a.IP[2] += byte(i) - err = v.vpp.AddInterfaceAddress(vppIfSwIfIndex, a) - if err != nil { - log.Errorf("Error adding address to data interface: %v", err) - } - } - return nil -} - -func (v *VppRunner) allocateStaticVRFs() error { - // Create all VRFs with a static ID that we use first so that we can - // then call AllocateVRF without risk of conflict - for _, ipFamily := range vpplink.IPFamilies { - err := v.vpp.AddVRF(common.PuntTableID, ipFamily.IsIP6, fmt.Sprintf("punt-table-%s", ipFamily.Str)) - if err != nil { - return errors.Wrapf(err, "Error creating punt vrf %s", ipFamily.Str) - } - err = v.vpp.AddVRF(common.PodVRFIndex, ipFamily.IsIP6, fmt.Sprintf("calico-pods-%s", ipFamily.Str)) - if err != nil { - return err - } - err = v.vpp.AddDefaultRouteViaTable(common.PodVRFIndex, common.DefaultVRFIndex, ipFamily.IsIP6) - if err != nil { - return err - } - } - return nil -} - // setupIPv6MulticastForHostTap configures mFIB entries to allow IPv6 multicast traffic // from the Linux host to pass through VPP. This is required for DHCPv6, NDP, and other // IPv6 protocols that use link-local multicast. // Without this configuration, packets arriving from the tap interface fail RPF checks // because the tap interface is not in the mFIB accept list. func (v *VppRunner) setupIPv6MulticastForHostTap(vrfID uint32, tapSwIfIndex uint32, uplinkSwIfIndex uint32) error { - log.Infof("Setting up IPv6 multicast forwarding for host tap in VRF %d", vrfID) + log.Debugf("Setting up IPv6 multicast forwarding for host tap in VRF %d", vrfID) // IPv6 multicast groups that need to be forwarded from the Linux host multicastGroups := []struct { - addr string + addr net.IP comment string }{ - {"ff02::1:2", "DHCPv6 All Relay Agents and Servers (REQUIRED for DHCPv6)"}, - {"ff02::1", "All Nodes (for NDP)"}, - {"ff02::2", "All Routers (for NDP/RA)"}, + {net.ParseIP("ff02::1:2"), "DHCPv6 All Relay Agents and Servers (REQUIRED for DHCPv6)"}, + {net.ParseIP("ff02::1"), "All Nodes (for NDP)"}, + {net.ParseIP("ff02::2"), "All Routers (for NDP/RA)"}, } for _, group := range multicastGroups { - groupIP := net.ParseIP(group.addr) - if groupIP == nil { - log.Warnf("Invalid multicast address: %s", group.addr) - continue - } - groupNet := &net.IPNet{ - IP: groupIP, + IP: group.addr, Mask: net.CIDRMask(128, 128), // /128 - specific group } - err := v.vpp.MRouteAddForHostMulticast(vrfID, groupNet, tapSwIfIndex, uplinkSwIfIndex) if err != nil { return errors.Wrapf(err, "cannot add mFIB route for %s (%s) in VRF %d", group.addr, group.comment, vrfID) } - - log.Infof("Added mFIB route for %s (%s) in VRF %d", group.addr, group.comment, vrfID) + log.Infof("added mFIB route for %s (%s) in VRF %d", group.addr, group.comment, vrfID) } return nil } -// Configure specific VRFs for a given tap to the host to handle broadcast / multicast traffic sent by the host -func (v *VppRunner) setupTapVRF(ifSpec *config.UplinkInterfaceSpec, ifState *config.LinuxInterfaceState, tapSwIfIndex uint32) (vrfs []uint32, err error) { - for _, ipFamily := range vpplink.IPFamilies { - vrfID, err := v.vpp.AllocateVRF(ipFamily.IsIP6, fmt.Sprintf("host-tap-%s-%s", ifSpec.InterfaceName, ipFamily.Str)) - if err != nil { - return []uint32{}, errors.Wrap(err, "Error allocating vrf for tap") - } - if ipFamily.IsIP4 { - // special route to forward broadcast from the host through the matching uplink - // useful for instance for DHCP DISCOVER pkts from the host - err = v.vpp.RouteAdd(&types.Route{ - Table: vrfID, - Dst: &net.IPNet{ - IP: net.IPv4bcast, - Mask: net.IPv4Mask(255, 255, 255, 255), - }, - Paths: []types.RoutePath{{ - Gw: net.ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"), - SwIfIndex: ifSpec.SwIfIndex, - }}, - }) - if err != nil { - log.Errorf("cannot add broadcast route in vpp: %v", err) - } - } else { - // Setup IPv6 multicast forwarding for the host - // This is required for DHCPv6 solicitations, NDP, and other link-local multicast - // Unlike IPv4, we cannot use a unicast route trick because ff02::/16 is multicast - // and must go through mFIB with proper RPF configuration - err = v.setupIPv6MulticastForHostTap(vrfID, tapSwIfIndex, ifSpec.SwIfIndex) - if err != nil { - return []uint32{}, errors.Wrap(err, "Error setting up IPv6 multicast forwarding") - } - } - - // default route in default table - err = v.vpp.AddDefaultRouteViaTable(vrfID, config.Info.PhysicalNets[ifSpec.PhysicalNetworkName].VrfID, ipFamily.IsIP6) - if err != nil { - return []uint32{}, errors.Wrapf(err, "error adding VRF %d default route via VRF %d", vrfID, config.Info.PhysicalNets[ifSpec.PhysicalNetworkName]) - } - // Set tap in this table - err = v.vpp.SetInterfaceVRF(tapSwIfIndex, vrfID, ipFamily.IsIP6) - if err != nil { - return []uint32{}, errors.Wrapf(err, "error setting vpp tap in vrf %d", vrfID) - } - vrfs = append(vrfs, vrfID) +// configureVppUplinkInterface configures one uplink interface in VPP +// and creates the corresponding tap in Linux +func (v *VppRunner) configureVppUplinkInterface(intf *config.VppManagerInterface) error { + err := intf.Driver.CreateMainVppInterface( + v.vpp, + vppProcess.Pid, + &intf.Spec, + ) + if err != nil { + return errors.Wrap(err, "Error creating uplink interface") } - // Configure addresses to enable ipv4 & ipv6 on the tap - for _, addr := range ifState.Addresses { - if addr.IP.IsLinkLocalUnicast() && !common.IsFullyQualified(addr.IPNet) && common.IsV6Cidr(addr.IPNet) { - log.Infof("Not adding address %s to data interface (vpp requires /128 link-local)", addr.String()) - continue - } else { - log.Infof("Adding address %s to tap interface", addr.String()) - } - // to max len cidr because we don't want the rest of the subnet to be considered as - // connected to that interface - // note that the role of these addresses is just to tell vpp to accept ip4 / ip6 packets on the tap - // we use these addresses as the safest option, because as they are configured on linux, linux - // will never send us packets with these addresses as destination - err = v.vpp.AddInterfaceAddress(tapSwIfIndex, common.ToMaxLenCIDR(addr.IP)) - if err != nil { - log.Errorf("Error adding address to tap interface: %v", err) - } + // Data interface configuration + err = v.vpp.Retry(2*time.Second, 10, v.vpp.InterfaceAdminUp, intf.Spec.SwIfIndex) + if err != nil { + return errors.Wrap(err, "Error setting uplink interface up") } - return vrfs, nil -} -// configureVppUplinkInterface configures one uplink interface in VPP -// and creates the corresponding tap in Linux -func (v *VppRunner) configureVppUplinkInterface( - uplinkDriver uplink.UplinkDriver, - ifState *config.LinuxInterfaceState, - ifSpec config.UplinkInterfaceSpec, -) (err error) { // Configure the physical network if we see it for the first time - if _, ok := config.Info.PhysicalNets[ifSpec.PhysicalNetworkName]; !ok { - err = v.AllocatePhysicalNetworkVRFs(ifSpec.PhysicalNetworkName) + if _, found := config.Info.PhysicalNets[intf.Spec.PhysicalNetworkName]; !found { + physicalNetwork, err := v.allocatePhysicalNetworkVRFs(intf.Spec.PhysicalNetworkName) if err != nil { - return err + return errors.Wrapf(err, "error creating physical network %s for uplink if %s", intf.Spec.PhysicalNetworkName, intf.Spec.InterfaceName) } + log.Infof("created VRF vrfId=%d podVrfId=%d", physicalNetwork.VrfID, physicalNetwork.PodVrfID) + config.Info.PhysicalNets[intf.Spec.PhysicalNetworkName] = *physicalNetwork } // Always enable GSO feature on data interface, only a tiny negative effect on perf if GSO is not // enabled on the taps or already done before an encap if *config.GetCalicoVppDebug().GSOEnabled { - err = v.vpp.EnableGSOFeature(ifSpec.SwIfIndex) + err = v.vpp.EnableGSOFeature(intf.Spec.SwIfIndex) if err != nil { - return errors.Wrap(err, "Error enabling GSO on uplink interface") + return errors.Wrapf(err, "error enabling GSO on uplink if %s", intf.Spec.InterfaceName) } } - uplinkMtu := vpplink.DefaultIntTo(ifSpec.Mtu, ifState.Mtu) - err = v.vpp.SetInterfaceMtu(ifSpec.SwIfIndex, uplinkMtu) + uplinkMtu := vpplink.DefaultIntTo(intf.Spec.Mtu, intf.State.Mtu) + err = v.vpp.SetInterfaceMtu(intf.Spec.SwIfIndex, uplinkMtu) if err != nil { - return errors.Wrapf(err, "Error setting %d MTU on uplink interface", uplinkMtu) + return errors.Wrapf(err, "error setting mtu=%d on uplink if %s", uplinkMtu, intf.Spec.InterfaceName) } - err = v.vpp.SetInterfaceRxMode(ifSpec.SwIfIndex, types.AllQueues, ifSpec.GetRxModeWithDefault(uplinkDriver.GetDefaultRxMode())) + err = v.vpp.SetInterfaceRxMode(intf.Spec.SwIfIndex, types.AllQueues, intf.Spec.GetRxModeWithDefault(intf.Driver.GetDefaultRxMode())) if err != nil { log.Warnf("%v", err) } - err = v.vpp.EnableInterfaceIP6(ifSpec.SwIfIndex) + err = v.vpp.EnableInterfaceIP6(intf.Spec.SwIfIndex) if err != nil { - return errors.Wrap(err, "Error enabling ipv6 on uplink interface") + return errors.Wrapf(err, "error enabling ip6 on uplink if %s", intf.Spec.InterfaceName) } - err = v.vpp.DisableIP6RouterAdvertisements(ifSpec.SwIfIndex) + err = v.vpp.DisableIP6RouterAdvertisements(intf.Spec.SwIfIndex) if err != nil { - return errors.Wrap(err, "Error disabling ipv6 RA on uplink interface") + return errors.Wrapf(err, "error disabling ip6 RA on uplink if %s", intf.Spec.InterfaceName) } - err = v.vpp.CnatEnableFeatures(ifSpec.SwIfIndex) + err = v.vpp.CnatEnableFeatures(intf.Spec.SwIfIndex) if err != nil { - return errors.Wrap(err, "Error configuring NAT on uplink interface") + return errors.Wrapf(err, "error enabling NAT on uplink if %s", intf.Spec.InterfaceName) } - if ifSpec.PhysicalNetworkName != "" { - for _, ipFamily := range vpplink.IPFamilies { - err = v.vpp.SetInterfaceVRF(ifSpec.SwIfIndex, config.Info.PhysicalNets[ifSpec.PhysicalNetworkName].VrfID, ipFamily.IsIP6) - if err != nil { - return errors.Wrapf(err, "error setting interface in vrf %d", config.Info.PhysicalNets[ifSpec.PhysicalNetworkName]) - } - } - value := config.Info.PhysicalNets[ifSpec.PhysicalNetworkName] - config.Info.PhysicalNets[ifSpec.PhysicalNetworkName] = config.PhysicalNetwork{ - VrfID: value.VrfID, - PodVrfID: value.PodVrfID, + for _, ipFamily := range vpplink.IPFamilies { + err = v.vpp.SetInterfaceVRF( + intf.Spec.SwIfIndex, + config.Info.PhysicalNets[intf.Spec.PhysicalNetworkName].VrfID, + ipFamily.IsIP6, + ) + if err != nil { + return errors.Wrapf(err, "error setting uplink if %s in vrf %d", intf.Spec.InterfaceName, config.Info.PhysicalNets[intf.Spec.PhysicalNetworkName]) } } - for _, addr := range ifState.Addresses { + for _, addr := range intf.State.Addresses { if addr.IP.IsLinkLocalUnicast() && !common.IsFullyQualified(addr.IPNet) && common.IsV6Cidr(addr.IPNet) { log.Infof("Not adding address %s to uplink interface (vpp requires /128 link-local)", addr.String()) continue } else { log.Infof("Adding address %s to uplink interface", addr.String()) } - err = v.vpp.AddInterfaceAddress(ifSpec.SwIfIndex, addr.IPNet) + err = v.vpp.AddInterfaceAddress(intf.Spec.SwIfIndex, addr.IPNet) if err != nil { log.Errorf("Error adding address to uplink interface: %v", err) } } - for _, route := range ifState.Routes { + for _, route := range intf.State.Routes { err = v.vpp.RouteAdd(&types.Route{ Dst: route.Dst, Paths: []types.RoutePath{{ Gw: route.Gw, - SwIfIndex: ifSpec.SwIfIndex, + SwIfIndex: intf.Spec.SwIfIndex, }}, }) if err != nil { - log.Errorf("cannot add route in vpp: %v", err) + log.Errorf("error adding route in vpp: %v", err) } } @@ -567,7 +366,7 @@ func (v *VppRunner) configureVppUplinkInterface( err = v.vpp.RouteAdd(&types.Route{ Paths: []types.RoutePath{{ Gw: defaultGW, - SwIfIndex: ifSpec.SwIfIndex, + SwIfIndex: intf.Spec.SwIfIndex, }}, }) if err != nil { @@ -575,15 +374,6 @@ func (v *VppRunner) configureVppUplinkInterface( } } - if ifSpec.IsMain { - if config.GetCalicoVppInitialConfig().ExtraAddrCount > 0 { - err = v.addExtraAddresses(ifState.Addresses, config.GetCalicoVppInitialConfig().ExtraAddrCount, ifSpec.SwIfIndex) - if err != nil { - log.Errorf("Cannot configure requested extra addresses: %v", err) - } - } - } - log.Infof("Creating Linux side interface") vpptap0Flags := types.TapFlagNone if *config.GetCalicoVppDebug().GSOEnabled { @@ -592,24 +382,104 @@ func (v *VppRunner) configureVppUplinkInterface( tapSwIfIndex, err := v.vpp.CreateTapV2(&types.TapV2{ GenericVppInterface: types.GenericVppInterface{ - HostInterfaceName: ifSpec.InterfaceName, + HostInterfaceName: intf.Spec.InterfaceName, RxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.RxQueueSize, TxQueueSize: config.GetCalicoVppInterfaces().VppHostTapSpec.TxQueueSize, - HardwareAddr: ifSpec.GetVppSideHardwareAddress(), + HardwareAddr: intf.Spec.GetVppSideHardwareAddress(), }, HostNamespace: "pid:1", // create tap in root netns - Tag: "host-" + ifSpec.InterfaceName, + Tag: "host-" + intf.Spec.InterfaceName, Flags: vpptap0Flags, HostMtu: uplinkMtu, - HostMacAddress: ifState.HardwareAddr, + HostMacAddress: intf.State.HardwareAddr, }) if err != nil { - return errors.Wrap(err, "Error creating tap") + return errors.Wrapf(err, "Error creating tap %s", intf.Spec.InterfaceName) + } + + // Configure specific VRFs for a given tap to the host to handle broadcast / multicast traffic sent by the host + for _, ipFamily := range vpplink.IPFamilies { + vrfID, err := v.vpp.AllocateVRF(ipFamily.IsIP6, fmt.Sprintf("host-tap-%s-%s", intf.Spec.InterfaceName, ipFamily.Str)) + if err != nil { + return errors.Wrapf(err, "error allocating %s vrf for if %s", ipFamily.Str, intf.Spec.InterfaceName) + } + // default route in default table + err = v.vpp.AddDefaultRouteViaTable(vrfID, config.Info.PhysicalNets[intf.Spec.PhysicalNetworkName].VrfID, ipFamily.IsIP6) + if err != nil { + return errors.Wrapf(err, "error adding %s VRF %d default route via VRF %d", ipFamily.Str, vrfID, config.Info.PhysicalNets[intf.Spec.PhysicalNetworkName]) + } + // Set tap in this table + err = v.vpp.SetInterfaceVRF(tapSwIfIndex, vrfID, ipFamily.IsIP6) + if err != nil { + return errors.Wrapf(err, "error setting vpp tap in %s vrf %d", ipFamily.Str, vrfID) + } + + if ipFamily.IsIP4 { + // special route to forward broadcast from the host through the matching uplink + // useful for instance for DHCP DISCOVER pkts from the host + err = v.vpp.RouteAdd(&types.Route{ + Table: vrfID, + Dst: &net.IPNet{ + IP: net.IPv4bcast, + Mask: net.IPv4Mask(255, 255, 255, 255), + }, + Paths: []types.RoutePath{{ + Gw: net.ParseIP("ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"), + SwIfIndex: intf.Spec.SwIfIndex, + }}, + }) + if err != nil { + return errors.Wrapf(err, "error add broadcast route for tap %d in v4 VRF %d", tapSwIfIndex, vrfID) + } + err = v.vpp.EnableArpProxy(tapSwIfIndex, vrfID) + if err != nil { + return errors.Wrapf(err, "error enabling ARP proxy for tap %d in v4 VRF %d", tapSwIfIndex, vrfID) + } + } else { + // Setup IPv6 multicast forwarding for the host + // This is required for DHCPv6 solicitations, NDP, and other link-local multicast + // Unlike IPv4, we cannot use a unicast route trick because ff02::/16 is multicast + // and must go through mFIB with proper RPF configuration + err = v.setupIPv6MulticastForHostTap(vrfID, tapSwIfIndex, intf.Spec.SwIfIndex) + if err != nil { + return errors.Wrapf(err, "error setting up IPv6 multicast forwarding tap %d in v6 vrf %d", tapSwIfIndex, vrfID) + } + } + for _, addr := range intf.State.Addresses { + if vpplink.IPFamilyFromIP(addr.IP) == ipFamily { + err = v.vpp.RouteAdd(&types.Route{ + Table: vrfID, + Dst: common.FullyQualified(addr.IP), + Paths: []types.RoutePath{{ + SwIfIndex: tapSwIfIndex, + }}, + }) + if err != nil { + return errors.Wrapf(err, "error add route from VPP to tap0 in VRF %d", vrfID) + } + err = v.vpp.AddNeighbor(&types.Neighbor{ + SwIfIndex: tapSwIfIndex, + IP: addr.IP, + HardwareAddr: intf.State.HardwareAddr, + Flags: types.IPNeighborStatic, + }) + if err != nil { + return errors.Wrapf(err, "error add static neighbor for tap0 in VRF %d", vrfID) + } + } + } + } + + err = v.vpp.EnableInterfaceIP6(tapSwIfIndex) + if err != nil { + return errors.Wrapf(err, "error enabling ip6 for tap %d", tapSwIfIndex) } - vrfs, err := v.setupTapVRF(&ifSpec, ifState, tapSwIfIndex) + // FIXME + _, cidr, _ := net.ParseCIDR("169.254.0.1/32") + err = v.vpp.AddInterfaceAddress(tapSwIfIndex, cidr) if err != nil { - return errors.Wrap(err, "error configuring VRF for tap") + return errors.Wrapf(err, "error enabling ip6 for tap %d", tapSwIfIndex) } // Always set this tap on worker 0 @@ -618,27 +488,42 @@ func (v *VppRunner) configureVppUplinkInterface( return errors.Wrap(err, "Error setting tap rx placement") } - err = v.vpp.SetInterfaceMtu(uint32(tapSwIfIndex), vpplink.CalicoVppMaxMTu) + err = v.vpp.SetInterfaceMtu(tapSwIfIndex, vpplink.CalicoVppMaxMTu) if err != nil { return errors.Wrapf(err, "Error setting %d MTU on tap interface", vpplink.CalicoVppMaxMTu) } - if ifState.Hasv6 { + if intf.State.Hasv6 { err = v.vpp.DisableIP6RouterAdvertisements(tapSwIfIndex) if err != nil { return errors.Wrap(err, "Error disabling ip6 RA on vpptap0") } } - err = v.configurePunt(tapSwIfIndex, *ifState) + err = v.vpp.AddNeighbor(&types.Neighbor{ + SwIfIndex: tapSwIfIndex, + IP: config.VppHostPuntFakeGatewayAddress, + HardwareAddr: intf.State.HardwareAddr, + Flags: types.IPNeighborStatic, + }) if err != nil { - return errors.Wrap(err, "Error adding redirect to tap") + return errors.Wrapf(err, "Error adding neighbor %s to tap", config.VppHostPuntFakeGatewayAddress) } - err = v.vpp.EnableArpProxy(tapSwIfIndex, vrfs[0 /* ip4 */]) - if err != nil { - return errors.Wrap(err, "Error enabling ARP proxy") + // In the punt table (where all punted traffics ends), route to the tap + for _, address := range intf.State.Addresses { + err = v.vpp.RouteAdd(&types.Route{ + Dst: address.IPNet, + Table: common.PuntTableID, + Paths: []types.RoutePath{{ + Gw: config.VppHostPuntFakeGatewayAddress, + SwIfIndex: tapSwIfIndex, + }}, + }) + if err != nil { + return errors.Wrapf(err, "error adding vpp side routes for interface") + } } - for _, addr := range ifState.Addresses { + for _, addr := range intf.State.Addresses { if addr.IP.To4() == nil { log.Infof("Adding ND proxy for address %s", addr.IP) err = v.vpp.EnableIP6NdProxy(tapSwIfIndex, addr.IP) @@ -676,12 +561,12 @@ func (v *VppRunner) configureVppUplinkInterface( } // Linux side tap setup - link, err := netlink.LinkByName(ifSpec.InterfaceName) + link, err := netlink.LinkByName(intf.Spec.InterfaceName) if err != nil { - return errors.Wrapf(err, "cannot find interface named %s", ifSpec.InterfaceName) + return errors.Wrapf(err, "cannot find interface named %s", intf.Spec.InterfaceName) } - fakeNextHopIP4, fakeNextHopIP6, err := v.configureLinuxTap(link, *ifState) + fakeNextHopIP4, fakeNextHopIP6, err := v.configureLinuxTap(link, *intf.State) if err != nil { return errors.Wrap(err, "Error configuring tap on linux side") } @@ -694,12 +579,12 @@ func (v *VppRunner) configureVppUplinkInterface( if config.Info.UplinkStatuses != nil { config.Info.UplinkStatuses[link.Attrs().Name] = config.UplinkStatus{ TapSwIfIndex: tapSwIfIndex, - SwIfIndex: ifSpec.SwIfIndex, + SwIfIndex: intf.Spec.SwIfIndex, Mtu: uplinkMtu, - PhysicalNetworkName: ifSpec.PhysicalNetworkName, + PhysicalNetworkName: intf.Spec.PhysicalNetworkName, LinkIndex: link.Attrs().Index, Name: link.Attrs().Name, - IsMain: ifSpec.IsMain, + IsMain: intf.Spec.IsMain, FakeNextHopIP4: fakeNextHopIP4, FakeNextHopIP6: fakeNextHopIP6, } @@ -708,9 +593,21 @@ func (v *VppRunner) configureVppUplinkInterface( } func (v *VppRunner) doVppGlobalConfiguration() (err error) { - err = v.allocateStaticVRFs() - if err != nil { - return errors.Wrap(err, "Error creating static VRFs in VPP") + // Create all VRFs with a static ID that we use first so that we can + // then call AllocateVRF without risk of conflict + for _, ipFamily := range vpplink.IPFamilies { + err := v.vpp.AddVRF(common.PuntTableID, ipFamily.IsIP6, fmt.Sprintf("punt-table-%s", ipFamily.Str)) + if err != nil { + return errors.Wrapf(err, "Error creating punt vrf %s", ipFamily.Str) + } + err = v.vpp.AddVRF(common.PodVRFIndex, ipFamily.IsIP6, fmt.Sprintf("calico-pods-%s", ipFamily.Str)) + if err != nil { + return err + } + err = v.vpp.AddDefaultRouteViaTable(common.PodVRFIndex, common.DefaultVRFIndex, ipFamily.IsIP6) + if err != nil { + return err + } } err = v.vpp.SetK8sSnatPolicy() @@ -734,6 +631,23 @@ func (v *VppRunner) doVppGlobalConfiguration() (err error) { return errors.Wrap(err, "error configuring v6 ip neighbors") } + for _, ipFamily := range vpplink.IPFamilies { + err = v.vpp.PuntRedirect(types.IPPuntRedirect{ + RxSwIfIndex: vpplink.InvalidID, + Paths: []types.RoutePath{{ + Table: common.PuntTableID, + SwIfIndex: types.InvalidID, + }}, + }, ipFamily.IsIP6) + if err != nil { + return errors.Wrapf(err, "Error configuring punt redirect") + } + + err = v.vpp.PuntAllL4(ipFamily.IsIP6) + if err != nil { + return errors.Wrapf(err, "Error configuring L4 punt") + } + } return nil } @@ -741,7 +655,7 @@ func (v *VppRunner) updateCalicoNode(ifState *config.LinuxInterfaceState) (err e var node, updated *oldv3.Node var client calicov3cli.Interface // TODO create if doesn't exist? need to be careful to do it atomically... and everyone else must as well. - for i := 0; i < 10; i++ { + for i := 0; i < 100; i++ { client, err = calicov3cli.NewFromEnv() if err != nil { return errors.Wrap(err, "Error creating calico client") @@ -800,68 +714,40 @@ func (v *VppRunner) updateCalicoNode(ifState *config.LinuxInterfaceState) (err e return errors.Wrap(err, "Error updating node") } -func (v *VppRunner) pingCalicoVpp() error { - dat, err := os.ReadFile(config.CalicoVppPidFile) - if err != nil { - if errors.Is(err, os.ErrNotExist) { - log.Infof("calico-vpp-pid file doesn't exist. Agent probably not started") - return nil - } - return errors.Wrapf(err, "Error reading %s", config.CalicoVppPidFile) - } - pid, err := strconv.ParseInt(strings.TrimSpace(string(dat[:])), 10, 32) +// allocatePhysicalNetworkVRFs allocates two VRFs for a phyiscal network +// and adds a route from the podVRF to the main VRF +func (v *VppRunner) allocatePhysicalNetworkVRFs(physicalNetworkName string) (*config.PhysicalNetwork, error) { + mainVrfID, err := v.vpp.AllocateVRF(false, fmt.Sprintf("physical-net-%s-ip4", physicalNetworkName)) if err != nil { - return errors.Wrapf(err, "Error parsing %s", dat) - } - err = syscall.Kill(int(pid), syscall.SIGUSR1) - if err != nil { - return errors.Wrapf(err, "Error kill -SIGUSR1 %d", int(pid)) - } - log.Infof("Did kill -SIGUSR1 %d", int(pid)) - return nil -} - -func (v *VppRunner) allInterfacesPhysical() bool { - for _, ifConf := range v.conf { - if ifConf.IsTunTap || ifConf.IsVeth { - return false - } + return nil, errors.Wrapf(err, "error allocating VRF physical-net-%s-ip4", physicalNetworkName) } - return true -} - -func (v *VppRunner) AllocatePhysicalNetworkVRFs(phyNet string) (err error) { - // for ip4 - mainVrfID, err := v.vpp.AllocateVRF(false, fmt.Sprintf("physical-net-%s-ip4", phyNet)) + podVrfID, err := v.vpp.AllocateVRF(false, fmt.Sprintf("calico-pods-%s-ip4", physicalNetworkName)) if err != nil { - return err + return nil, errors.Wrapf(err, "error allocating VRF calico-pods-%s-ip4", physicalNetworkName) } - podVrfID, err := v.vpp.AllocateVRF(false, fmt.Sprintf("calico-pods-%s-ip4", phyNet)) + err = v.vpp.AddVRF(mainVrfID, true, fmt.Sprintf("physical-net-%s-ip6", physicalNetworkName)) if err != nil { - return err + return nil, errors.Wrapf(err, "error allocating VRF physical-net-%s-ip6", physicalNetworkName) } - // for ip6, use same vrfID as ip4 - err = v.vpp.AddVRF(mainVrfID, true, fmt.Sprintf("physical-net-%s-ip6", phyNet)) + err = v.vpp.AddVRF(podVrfID, true, fmt.Sprintf("calico-pods-%s-ip6", physicalNetworkName)) if err != nil { - return err - } - err = v.vpp.AddVRF(podVrfID, true, fmt.Sprintf("calico-pods-%s-ip6", phyNet)) - if err != nil { - return err + return nil, errors.Wrapf(err, "error allocating VRF calico-pods-%s-ip6", physicalNetworkName) } for _, ipFamily := range vpplink.IPFamilies { err = v.vpp.AddDefaultRouteViaTable(podVrfID, mainVrfID, ipFamily.IsIP6) if err != nil { - return err + return nil, errors.Wrapf(err, "error adding default %s route from podVRF to mainVRF for physical network %s", ipFamily.Str, physicalNetworkName) } } - config.Info.PhysicalNets[phyNet] = config.PhysicalNetwork{VrfID: mainVrfID, PodVrfID: podVrfID} - return nil + return &config.PhysicalNetwork{ + VrfID: mainVrfID, + PodVrfID: podVrfID, + }, nil } // Returns VPP exit code func (v *VppRunner) runVpp() (err error) { - if !v.allInterfacesPhysical() { // use separate net namespace because linux deletes these interfaces when ns is deleted + if !v.params.AllInterfacesPhysical() { // use separate net namespace because linux deletes these interfaces when ns is deleted if ns.IsNSorErr(utils.GetnetnsPath(config.VppNetnsName)) != nil { _, err = utils.NewNS(config.VppNetnsName) if err != nil { @@ -904,10 +790,9 @@ func (v *VppRunner) runVpp() (err error) { vppProcess = vppCmd.Process } - /** - * From this point it is very important that every exit - * path calls restoreConfiguration after vpp exits */ - defer v.restoreConfiguration(v.allInterfacesPhysical()) + // From this point it is very important that every exit + // path calls restoreConfiguration after vpp exits + defer v.restoreConfiguration() log.Infof("VPP started [PID %d]", vppProcess.Pid) runningCond.Broadcast() @@ -934,38 +819,16 @@ func (v *VppRunner) runVpp() (err error) { // add main network that has the default VRF config.Info.PhysicalNets[config.DefaultPhysicalNetworkName] = config.PhysicalNetwork{VrfID: common.DefaultVRFIndex, PodVrfID: common.PodVRFIndex} - err = v.configureGlobalPunt() - if err != nil { - return errors.Wrap(err, "Error adding redirect to tap") - } - for idx := 0; idx < len(v.params.UplinksSpecs); idx++ { - err := v.uplinkDriver[idx].CreateMainVppInterface(vpp, vppProcess.Pid, &v.params.UplinksSpecs[idx]) + for interfaceName, intf := range v.params.Interfaces { + err = v.configureVppUplinkInterface(intf) if err != nil { - terminateVpp("Error creating uplink interface %s: %v", v.params.UplinksSpecs[idx].InterfaceName, err) - v.vpp.Close() + terminateVpp("Error configuring VPP interface %s %v", interfaceName, err) <-vppDeadChan - return errors.Wrap(err, "Error creating uplink interface") - } - - // Data interface configuration - err = v.vpp.Retry(2*time.Second, 10, v.vpp.InterfaceAdminUp, v.params.UplinksSpecs[idx].SwIfIndex) - if err != nil { - terminateVpp("Error setting uplink interface up: %v", err) - v.vpp.Close() - <-vppDeadChan - return errors.Wrap(err, "Error setting uplink interface up") - } - - err = v.configureVppUplinkInterface(v.uplinkDriver[idx], v.conf[idx], v.params.UplinksSpecs[idx]) - - if err != nil { - terminateVpp("Error configuring VPP: %v", err) - <-vppDeadChan - return errors.Wrap(err, "Error configuring VPP") + return errors.Wrapf(err, "Error configuring VPP interface %s", interfaceName) } } // Update the Calico node with the IP address actually configured on VPP - err = v.updateCalicoNode(v.conf[0]) + err = v.updateCalicoNode(v.params.InterfacesById[0].State) if err != nil { terminateVpp("Error updating Calico node (SIGINT %d): %v", vppProcess.Pid, err) <-vppDeadChan @@ -977,33 +840,28 @@ func (v *VppRunner) runVpp() (err error) { if err != nil { log.Errorf("Error writing vpp manager file: %v", err) } - var t tomb.Tomb // close vpp as we do not program v.vpp.Close() - networkHook.ExecuteWithUserScript(hooks.HookVppRunning, config.HookScriptVppRunning, v.params) + networkHook.ExecuteWithUserScript(hooks.HookVppRunning, config.HookScriptVppRunning) <-vppDeadChan log.Infof("VPP Exited: status %v", err) - err = t.Killf("Vpp exited, stopping watchers") - if err != nil { - log.Errorf("Error Killf vpp: %v", err) - } return nil } -func (v *VppRunner) restoreConfiguration(allInterfacesPhysical bool) { +func (v *VppRunner) restoreConfiguration() { log.Infof("Restoring configuration") err := utils.ClearVppManagerFiles() if err != nil { log.Errorf("Error clearing vpp manager files: %v", err) } - for idx := range v.params.UplinksSpecs { - v.uplinkDriver[idx].RestoreLinux(allInterfacesPhysical) + for _, intf := range v.params.Interfaces { + intf.Driver.RestoreLinux() } - err = v.pingCalicoVpp() + err = utils.PingCalicoVpp() if err != nil { log.Errorf("Error pinging calico-vpp: %v", err) }