Vpp-manager refactoring

Signed-off-by: Nathan Skrzypczak <[email protected]>

Author: sknat

config/config.go

@@ -162,13 +162,13 @@ func RunHook(hookScript *string, hookName string, params *VppManagerParams, log
 	if *hookScript == "" {
 		return
 	}
-	template, err := TemplateScriptReplace(*hookScript, params, nil)
+	template, err := TemplateScriptReplace(*hookScript, params)
 	if err != nil {
 		log.Warnf("Running hook %s errored with %s", hookName, err)
 		return
 	}
 
-	cmd := exec.Command("/bin/bash", "-c", template, hookName, params.UplinksSpecs[0].InterfaceName)
+	cmd := exec.Command("/bin/bash", "-c", template, hookName, params.InterfacesById[0].Spec.InterfaceName)
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stderr
 	err = cmd.Run()
@@ -428,8 +428,6 @@ type CalicoVppInitialConfigConfigType struct { //out of agent and vppmanager
 	// CorePattern is the pattern to use for VPP corefiles.
 	// Usually "/var/lib/vpp/vppcore.%e.%p"
 	CorePattern      string `json:"corePattern"`
-	ExtraAddrCount   int    `json:"extraAddrCount"`
-	IfConfigSavePath string `json:"ifConfigSavePath"`
 	// DefaultGWs Comma separated list of IPs to be
 	// configured in VPP as default GW
 	DefaultGWs string `json:"defaultGWs"`
@@ -619,9 +617,35 @@ const (
 	VfioUnsafeNoIommuModeDISABLED UnsafeNoIommuMode = "disabled"
 )
 
+type UplinkDriver interface {
+	PreconfigureLinux() error
+	CreateMainVppInterface(vpp *vpplink.VppLink, vppPid int, uplinkSpec *UplinkInterfaceSpec) error
+	RestoreLinux()
+	IsSupported(warn bool) bool
+	GetName() string
+	UpdateVppConfigFile(template string) string
+	GetDefaultRxMode() types.RxMode
+}
+
+type VppManagerInterface struct {
+	Spec UplinkInterfaceSpec
+	State *LinuxInterfaceState
+	Driver UplinkDriver
+}
+
+func (p *VppManagerParams) AllInterfacesPhysical() bool {
+	for _, intf := range p.Interfaces {
+		if intf.State.IsTunTap || intf.State.IsVeth {
+			return false
+		}
+	}
+	return true
+}
+
 type VppManagerParams struct {
-	UplinksSpecs []UplinkInterfaceSpec
-	/* Capabilities */
+	Interfaces map[string]*VppManagerInterface
+	InterfacesById []*VppManagerInterface
+	// Capabilities
 	LoadedDrivers                      map[string]bool
 	KernelVersion                      *KernelVersion
 	AvailableHugePages                 int
@@ -744,23 +768,25 @@ func getCpusetCPU() (string, error) {
 	return regexp.MustCompile("[,-]").Split(cpusetCPU, 2)[0], nil
 }
 
-func TemplateScriptReplace(input string, params *VppManagerParams, conf []*LinuxInterfaceState) (template string, err error) {
+func TemplateScriptReplace(input string, params *VppManagerParams) (template string, err error) {
 	template = input
-	if conf != nil {
-		/* We might template scripts before reading interface conf */
-		template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", conf[0].PciID)
-		for i, ifcConf := range conf {
-			template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(i)+"__", ifcConf.PciID)
-		}
+	if len(params.Interfaces) > 0 {
+		// We might template scripts before reading interface conf
+		template = strings.ReplaceAll(template, "__PCI_DEVICE_ID__", params.InterfacesById[0].State.PciID)
+	}
+	for idx, intf := range params.InterfacesById {
+		template = strings.ReplaceAll(template, "__PCI_DEVICE_ID_"+strconv.Itoa(idx)+"__", intf.State.PciID)
 	}
 	vppcpu, err := getCpusetCPU()
 	if err != nil {
 		return "", err
 	}
 	template = strings.ReplaceAll(template, "__CPUSET_CPUS_FIRST__", vppcpu)
-	template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.UplinksSpecs[0].InterfaceName)
-	for i, ifc := range params.UplinksSpecs {
-		template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", i)+"__", ifc.InterfaceName)
+	if len(params.Interfaces) > 0 {
+		template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF__", params.InterfacesById[0].Spec.InterfaceName)
+	}
+	for idx, intf := range params.InterfacesById {
+		template = strings.ReplaceAll(template, "__VPP_DATAPLANE_IF_"+fmt.Sprintf("%d", idx)+"__", intf.Spec.InterfaceName)
 	}
 	for key, value := range params.NodeAnnotations {
 		template = strings.ReplaceAll(template, fmt.Sprintf("__NODE_ANNOTATION:%s__", key), value)

vpp-manager/hooks/network_manager_hook.go

@@ -52,9 +52,9 @@ type SystemType struct {
 
 // NetworkManagerHook manages network configuration during VPP lifecycle
 type NetworkManagerHook struct {
-	interfaceNames []string
-	systemType     SystemType
-	log            *logrus.Logger
+	vppManagerParams *config.VppManagerParams
+	systemType       SystemType
+	log              *logrus.Logger
 }
 
 // chrootCommand creates a command that will be executed in the host namespace
@@ -151,21 +151,16 @@ func (h *NetworkManagerHook) restartService(serviceName string) error {
 }
 
 // NewNetworkManagerHook creates a new NetworkManagerHook instance
-func NewNetworkManagerHook(log *logrus.Logger) *NetworkManagerHook {
+func NewNetworkManagerHook(log *logrus.Logger, vppManagerParams *config.VppManagerParams) *NetworkManagerHook {
 	hook := &NetworkManagerHook{
-		log: log,
+		log:              log,
+		vppManagerParams: vppManagerParams,
 	}
 
 	hook.detectSystem()
 	return hook
 }
 
-// SetInterfaceNames updates the interface names of NetworkManagerHook instance
-func (h *NetworkManagerHook) SetInterfaceNames(interfaceNames []string) {
-	h.interfaceNames = interfaceNames
-	h.log.Infof("NetworkManagerHook: Interface names updated to %v", interfaceNames)
-}
-
 // fixDNS modifies NetworkManager configuration to disable DNS management
 func (h *NetworkManagerHook) fixDNS() error {
 	if !h.systemType.HasNetworkManager {
@@ -500,7 +495,7 @@ func (h *NetworkManagerHook) beforeVppRun() error {
 	}
 
 	// Save network file for AWS systemd-networkd for each interface
-	for _, interfaceName := range h.interfaceNames {
+	for interfaceName := range h.vppManagerParams.Interfaces {
 		err = h.saveNetworkFile(interfaceName)
 		if err != nil {
 			return err
@@ -519,7 +514,7 @@ func (h *NetworkManagerHook) vppRunning() error {
 	}
 
 	// Tweak network file for AWS systemd-networkd for each interface
-	for _, interfaceName := range h.interfaceNames {
+	for interfaceName := range h.vppManagerParams.Interfaces {
 		err = h.tweakNetworkFile(interfaceName)
 		if err != nil {
 			return err
@@ -538,7 +533,7 @@ func (h *NetworkManagerHook) vppDoneOk() error {
 	}
 
 	// Remove the tweaked network file for AWS systemd-networkd for each interface
-	for _, interfaceName := range h.interfaceNames {
+	for interfaceName := range h.vppManagerParams.Interfaces {
 		err = h.removeTweakedNetworkFile(interfaceName)
 		if err != nil {
 			return err
@@ -566,7 +561,7 @@ func (h *NetworkManagerHook) Execute(hookPoint HookPoint) error {
 		return nil
 	}
 
-	h.log.Infof("NetworkManagerHook: Executing %s for interfaces %v", hookPoint, h.interfaceNames)
+	h.log.Infof("NetworkManagerHook: Executing %s for all interfaces", hookPoint)
 
 	var err error
 	switch hookPoint {
@@ -616,11 +611,10 @@ func (h *NetworkManagerHook) useDefaultHookFallback(userHook *string) bool {
 // 2. If native hooks enabled -> run native Go hook
 // 3. If native hooks disabled & no user script -> fallback to default_hook.sh
 func (h *NetworkManagerHook) ExecuteWithUserScript(hookPoint HookPoint,
-	hookScript *string,
-	params *config.VppManagerParams) {
+	hookScript *string) {
 	// Check if user script is configured (highest priority - overrides everything)
 	if hookScript != nil && *hookScript != "" {
-		config.RunHook(hookScript, string(hookPoint), params, h.log)
+		config.RunHook(hookScript, string(hookPoint), h.vppManagerParams, h.log)
 		return
 	}
 
@@ -636,6 +630,6 @@ func (h *NetworkManagerHook) ExecuteWithUserScript(hookPoint HookPoint,
 	// Fallback to default_hook.sh when native hooks are disabled
 	if h.useDefaultHookFallback(hookScript) {
 		h.log.Infof("Native hooks disabled, falling back to default_hook.sh for %s", hookPoint)
-		config.RunHook(&config.DefaultHookScript, string(hookPoint), params, h.log)
+		config.RunHook(&config.DefaultHookScript, string(hookPoint), h.vppManagerParams, h.log)
 	}
 }

vpp-manager/main.go

@@ -165,8 +165,8 @@ func main() {
 
 	/* Initialize native Go NewNetworkManagerHook with empty interface names
 	 * We will update this later once we have the actual interface names */
-	networkHook = hooks.NewNetworkManagerHook(log)
-	networkHook.ExecuteWithUserScript(hooks.HookBeforeIfRead, config.HookScriptBeforeIfRead, params)
+	networkHook = hooks.NewNetworkManagerHook(log, params)
+	networkHook.ExecuteWithUserScript(hooks.HookBeforeIfRead, config.HookScriptBeforeIfRead)
 
 	err = utils.ClearVppManagerFiles()
 	if err != nil {
@@ -183,44 +183,31 @@ func main() {
 		log.Errorf("Error raising memlock limit, VPP may fail to start: %v", err)
 	}
 
-	confs, err := startup.GetInterfaceConfig(params)
-	if err != nil {
-		log.Fatalf("Error getting initial interface configuration: %s", err)
-	}
-
-	/* Update native Go NewNetworkManagerHook with all interface names */
-	if len(params.UplinksSpecs) > 0 {
-		interfaceNames := make([]string, len(params.UplinksSpecs))
-		for i, spec := range params.UplinksSpecs {
-			interfaceNames[i] = spec.InterfaceName
-		}
-		networkHook.SetInterfaceNames(interfaceNames)
-	}
-
 	runningCond = sync.NewCond(&sync.Mutex{})
 	go handleSignals()
 
 	ctx, cancel := context.WithCancel(context.Background())
 	defer cancel()
 	config.HandleUsr2Signal(ctx, log.WithFields(logrus.Fields{"component": "sighdlr"}))
 
-	startup.PrintVppManagerConfig(params, confs)
-
-	runner := NewVPPRunner(params, confs)
+	startup.PrintVppManagerConfig(params)
 
 	makeNewVPPIndex()
 
-	if len(params.UplinksSpecs) == 1 && params.UplinksSpecs[0].VppDriver == "" {
-		for _, driver := range uplink.SupportedUplinkDrivers(params, confs[0], &params.UplinksSpecs[0]) {
+	if len(params.Interfaces) == 1 && params.InterfacesById[0].Spec.VppDriver == "" {
+		intf := params.InterfacesById[0]
+		for _, driver := range uplink.SupportedUplinkDrivers(params, intf) {
 			err := utils.CleanupCoreFiles(config.GetCalicoVppInitialConfig().CorePattern, maxCoreFiles)
 			if err != nil {
 				log.Errorf("CleanupCoreFiles errored %s", err)
 			}
+			intf.Driver = driver
 
 			internalKill = false
-			err = runner.Run([]uplink.UplinkDriver{driver})
+			runner := NewVPPRunner(params)
+			err = runner.Run()
 			if err != nil {
-				networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored, params)
+				networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored)
 				log.Errorf("VPP(%s) run failed with %s", driver.GetName(), err)
 			}
 			if vppProcess != nil && !internalKill {
@@ -236,15 +223,10 @@ func main() {
 			log.Errorf("CleanupCoreFiles errored %s", err)
 		}
 
-		var drivers []uplink.UplinkDriver
-		for idx := 0; idx < len(params.UplinksSpecs); idx++ {
-			drivers = append(drivers, uplink.NewUplinkDriver(params.UplinksSpecs[idx].VppDriver,
-				params, confs[idx], &params.UplinksSpecs[idx]))
-		}
-
-		err = runner.Run(drivers)
+		runner := NewVPPRunner(params)
+		err = runner.Run()
 		if err != nil {
-			networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored, params)
+			networkHook.ExecuteWithUserScript(hooks.HookVppErrored, config.HookScriptVppErrored)
 			log.Errorf("VPP run failed with %v", err)
 		}
 

vpp-manager/startup/interface_config.go

@@ -16,10 +16,7 @@
 package startup
 
 import (
-	"encoding/gob"
-	"fmt"
 	"net"
-	"os"
 
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
@@ -30,56 +27,6 @@ import (
 	"github.com/projectcalico/vpp-dataplane/v3/vpplink"
 )
 
-func GetInterfaceConfig(params *config.VppManagerParams) (conf []*config.LinuxInterfaceState, err error) {
-	errs := []error{}
-	conf = []*config.LinuxInterfaceState{}
-	for _, ifSpec := range params.UplinksSpecs {
-		configuration, err := loadInterfaceConfigFromLinux(ifSpec)
-		errs = append(errs, err)
-		conf = append(conf, configuration)
-	}
-	allLoaded := true
-	for i := range errs {
-		if errs[i] != nil {
-			allLoaded = false
-			log.Warnf("Could not load config from linux (%v)", errs[i])
-		}
-	}
-	if allLoaded {
-		err = saveConfig(params, conf)
-		if err != nil {
-			log.Warnf("Could not save interface config: %v", err)
-		}
-	} else {
-		// Loading config failed, try loading from save file
-		confFile, err2 := loadInterfaceConfigFromFile(params)
-		if err2 != nil {
-			return nil, err2
-		}
-		// If loaded from file replace non loaded interface configs
-		for i := range conf {
-			if conf[i] == nil {
-				for j := range confFile {
-					if confFile[j].InterfaceName == params.UplinksSpecs[i].InterfaceName {
-						conf[i] = confFile[j]
-					}
-				}
-			}
-		}
-		for i := range conf {
-			if conf[i] == nil {
-				return nil, fmt.Errorf("interface configs not found")
-			}
-		}
-		log.Infof("Loaded config. Interfaces marked as down since loading config from linux failed.")
-		// This ensures we don't try to set the interface down in runVpp()
-		for _, config := range conf {
-			config.IsUp = false
-		}
-	}
-	return conf, nil
-}
-
 func loadInterfaceConfigFromLinux(ifSpec config.UplinkInterfaceSpec) (*config.LinuxInterfaceState, error) {
 	conf := config.LinuxInterfaceState{}
 	link, err := netlink.LinkByName(ifSpec.InterfaceName)
@@ -146,53 +93,3 @@ func getNodeAddress(conf *config.LinuxInterfaceState, isV6 bool) string {
 	}
 	return ""
 }
-
-func clearSavedConfig(params *config.VppManagerParams) {
-	if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" {
-		return
-	}
-	err := os.Remove(config.GetCalicoVppInitialConfig().IfConfigSavePath)
-	if err != nil {
-		log.Warnf("could not delete saved interface config: %v", err)
-	}
-}
-
-func saveConfig(params *config.VppManagerParams, conf []*config.LinuxInterfaceState) error {
-	if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" {
-		return nil
-	}
-	file, err := os.Create(config.GetCalicoVppInitialConfig().IfConfigSavePath)
-	if err != nil {
-		return errors.Wrap(err, "error opening save file")
-	}
-	enc := gob.NewEncoder(file)
-	err = enc.Encode(conf)
-	if err != nil {
-		file.Close()
-		clearSavedConfig(params)
-		return errors.Wrap(err, "error encoding data")
-	}
-	err = file.Close()
-	if err != nil {
-		return errors.Wrap(err, "error closing file")
-	}
-	return nil
-}
-
-func loadInterfaceConfigFromFile(params *config.VppManagerParams) ([]*config.LinuxInterfaceState, error) {
-	conf := []*config.LinuxInterfaceState{}
-	if config.GetCalicoVppInitialConfig().IfConfigSavePath == "" {
-		return nil, fmt.Errorf("interface config save file not configured")
-	}
-	file, err := os.Open(config.GetCalicoVppInitialConfig().IfConfigSavePath)
-	if err != nil {
-		return nil, errors.Wrap(err, "error opening save file")
-	}
-	dec := gob.NewDecoder(file)
-	err = dec.Decode(&conf)
-	if err != nil {
-		return nil, errors.Wrap(err, "decode error")
-	}
-	file.Close()
-	return conf, nil
-}