diff --git a/cmd/nfd-worker/main.go b/cmd/nfd-worker/main.go index 120db537ab..ce8c2812f3 100644 --- a/cmd/nfd-worker/main.go +++ b/cmd/nfd-worker/main.go @@ -106,6 +106,8 @@ func initFlags(flagset *flag.FlagSet) (*worker.Args, *worker.ConfigOverrideArgs) "Config file to use.") flagset.StringVar(&args.Kubeconfig, "kubeconfig", "", "Kubeconfig to use") + flagset.StringVar(&args.KubeletConfigPath, "kubelet-config-path", "/var/lib/kubelet/config.yaml", + "Path to the kubelet configuration file") flagset.BoolVar(&args.Oneshot, "oneshot", false, "Do not publish feature labels") flagset.IntVar(&args.Port, "port", 8080, diff --git a/deployment/components/common/worker-mounts.yaml b/deployment/components/common/worker-mounts.yaml index 8b64f271ad..9ae517c7ce 100644 --- a/deployment/components/common/worker-mounts.yaml +++ b/deployment/components/common/worker-mounts.yaml @@ -22,6 +22,9 @@ - name: features-d hostPath: path: "/etc/kubernetes/node-feature-discovery/features.d/" + - name: kubelet-dir + hostPath: + path: "/var/lib/kubelet/config.yaml" - name: nfd-worker-conf configMap: name: nfd-worker-conf @@ -50,6 +53,9 @@ - name: features-d mountPath: "/etc/kubernetes/node-feature-discovery/features.d/" readOnly: true + - name: kubelet-dir + mountPath: "/var/lib/kubelet/config.yaml" + readOnly: true - name: nfd-worker-conf mountPath: "/etc/kubernetes/node-feature-discovery" readOnly: true diff --git a/docs/reference/worker-commandline-reference.md b/docs/reference/worker-commandline-reference.md index 842c859903..92ffe267d8 100644 --- a/docs/reference/worker-commandline-reference.md +++ b/docs/reference/worker-commandline-reference.md @@ -84,6 +84,24 @@ Example: nfd-worker -kubeconfig ${HOME}/.kube/config ``` +### -kubelet-config-path + +The `-kubeconfig-path` flag specifies the absolute path to the kubelet configuration +file on the node. This configuration is primarily used by NFD to determine the +swap behavior set on the node. Kubernetes currently supports two types of swap +behavior `NoSwap` and `LimitedSwap`. It is important to note that even if swap is +enabled at the system level, a setting of `NoSwap` in the kubelet configuration +means that Kubernetes workloads will not be permitted to use swap space—although +non-Kubernetes processes on the node may still do so. + +Default: `/var/lib/kubelet/config.yaml` + +Example: + +```bash +nfd-worker -kubeconfig-path /var/lib/kubelet/config.yaml +``` + ### -feature-sources The `-feature-sources` flag specifies a comma-separated list of enabled feature diff --git a/docs/reference/worker-configuration-reference.md b/docs/reference/worker-configuration-reference.md index e51353ee00..f674b0bd36 100644 --- a/docs/reference/worker-configuration-reference.md +++ b/docs/reference/worker-configuration-reference.md @@ -245,6 +245,18 @@ Comma-separated list of `pattern=N` settings for file-filtered logging. Default: *empty* +### core.kubeletConfigPath + +Specifies the absolute path to the kubelet configuration file on the node. This +configuration is primarily used by NFD to determine the swap behavior set on +the node. Kubernetes currently supports two types of swap behavior `NoSwap` and +`LimitedSwap`. It is important to note that even if swap is enabled at the +system level, a setting of `NoSwap` in the kubelet configuration means that +Kubernetes workloads will not be permitted to use swap space—although +non-Kubernetes processes on the node may still do so. + +Default: `/var/lib/kubelet/config.yaml` + ## sources The `sources` section contains feature source specific configuration parameters. diff --git a/pkg/nfd-worker/nfd-worker.go b/pkg/nfd-worker/nfd-worker.go index f54a901ec5..0fd87c9378 100644 --- a/pkg/nfd-worker/nfd-worker.go +++ b/pkg/nfd-worker/nfd-worker.go @@ -57,6 +57,7 @@ import ( _ "sigs.k8s.io/node-feature-discovery/source/kernel" _ "sigs.k8s.io/node-feature-discovery/source/local" _ "sigs.k8s.io/node-feature-discovery/source/memory" + memory "sigs.k8s.io/node-feature-discovery/source/memory" _ "sigs.k8s.io/node-feature-discovery/source/network" _ "sigs.k8s.io/node-feature-discovery/source/pci" _ "sigs.k8s.io/node-feature-discovery/source/storage" @@ -76,6 +77,14 @@ type NFDConfig struct { Sources sourcesConfig } +func configureKubeletConfigPath(kubeletConfigPath string) error { + if kubeletConfigPath == "" { + return fmt.Errorf("kubelet config path is empty, using default: '/var/lib/kubelet/config.yaml'") + } + memory.SetKubeletConfigPath(kubeletConfigPath) + return nil +} + type coreConfig struct { Klog klogutils.KlogConfigOpts LabelWhiteList utils.RegexpVal @@ -94,13 +103,14 @@ type Labels map[string]string // Args are the command line arguments of NfdWorker. type Args struct { - ConfigFile string - Klog map[string]*utils.KlogFlagVal - Kubeconfig string - Oneshot bool - Options string - Port int - NoOwnerRefs bool + ConfigFile string + Klog map[string]*utils.KlogFlagVal + Kubeconfig string + Oneshot bool + Options string + Port int + NoOwnerRefs bool + KubeletConfigPath string Overrides ConfigOverrideArgs } @@ -312,6 +322,10 @@ func (w *nfdWorker) Run() error { httpMux.Handle("/metrics", promhttp.HandlerFor(promRegistry, promhttp.HandlerOpts{})) registerVersion(version.Get()) + if err := configureKubeletConfigPath(w.args.KubeletConfigPath); err != nil { + klog.ErrorS(err, "failed to configure kubelet config path") + } + err = w.runFeatureDiscovery() if err != nil { return err diff --git a/pkg/utils/kubeconf/kubelet_config_file.go b/pkg/utils/kubeconf/kubelet_config_file.go index f863090b08..fa55233215 100644 --- a/pkg/utils/kubeconf/kubelet_config_file.go +++ b/pkg/utils/kubeconf/kubelet_config_file.go @@ -18,11 +18,15 @@ package kubeconf import ( "fmt" + "os" + "github.com/pkg/errors" + kubeletconfig "k8s.io/kubelet/config/v1beta1" kubeletconfigv1beta1 "k8s.io/kubelet/config/v1beta1" kubeletconfigscheme "k8s.io/kubernetes/pkg/kubelet/apis/config/scheme" "k8s.io/kubernetes/pkg/kubelet/kubeletconfig/configfiles" utilfs "k8s.io/kubernetes/pkg/util/filesystem" + "sigs.k8s.io/yaml" ) // GetKubeletConfigFromLocalFile returns KubeletConfiguration loaded from the node local config @@ -54,3 +58,23 @@ func GetKubeletConfigFromLocalFile(kubeletConfigPath string) (*kubeletconfigv1be return kubeletConfig, nil } + +// ReadKubeletConfig reads and unmarshals a kubelet configuration file from the specified file. +func ReadKubeletConfig(kubeletFile string) (*kubeletconfig.KubeletConfiguration, error) { + _, err := os.Stat(kubeletFile) + if os.IsNotExist(err) { + return nil, fmt.Errorf("kubelet config file %s does not exist", kubeletFile) + } + + data, err := os.ReadFile(kubeletFile) + if err != nil { + return nil, errors.Wrapf(err, "failed to read kubelet configuration file %q", kubeletFile) + } + + var config kubeletconfig.KubeletConfiguration + if err := yaml.Unmarshal(data, &config); err != nil { + return nil, errors.Wrapf(err, "could not parse kubelet configuration file %q", kubeletFile) + } + + return &config, nil +} diff --git a/source/memory/memory.go b/source/memory/memory.go index 68ef0c741f..ef586b7e3d 100644 --- a/source/memory/memory.go +++ b/source/memory/memory.go @@ -31,6 +31,7 @@ import ( nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1" "sigs.k8s.io/node-feature-discovery/pkg/utils" "sigs.k8s.io/node-feature-discovery/pkg/utils/hostpath" + "sigs.k8s.io/node-feature-discovery/pkg/utils/kubeconf" "sigs.k8s.io/node-feature-discovery/source" ) @@ -54,13 +55,25 @@ type memorySource struct { features *nfdv1alpha1.Features } +// KubeletConfigPath holds the path to the kubelet configuration file. +type KubeletConfigPath struct { + ConfigFilePath string +} + // Singleton source instance var ( - src memorySource - _ source.FeatureSource = &src - _ source.LabelSource = &src + src memorySource + _ source.FeatureSource = &src + _ source.LabelSource = &src + defaultSwapBehavior = "NoSwap" ) +var kubelet = KubeletConfigPath{} + +func SetKubeletConfigPath(path string) { + kubelet.ConfigFilePath = path +} + // Name returns an identifier string for this feature source. func (s *memorySource) Name() string { return Name } @@ -80,6 +93,7 @@ func (s *memorySource) GetLabels() (source.FeatureLabels, error) { // Swap if isSwap, ok := features.Attributes[SwapFeature].Elements["enabled"]; ok && isSwap == "true" { labels["swap"] = true + labels["swap.behavior"] = features.Attributes[SwapFeature].Elements["behavior"] } // NVDIMM @@ -107,11 +121,19 @@ func (s *memorySource) Discover() error { s.features.Attributes[NumaFeature] = nfdv1alpha1.AttributeFeatureSet{Elements: numa} } - // Detect Swap + // Detect Swap and Swap Behavior if swap, err := detectSwap(); err != nil { klog.ErrorS(err, "failed to detect Swap nodes") } else { s.features.Attributes[SwapFeature] = nfdv1alpha1.AttributeFeatureSet{Elements: swap} + swapBehavior, err := detectSwapBehavior(kubelet.ConfigFilePath) + if err != nil { + klog.V(3).ErrorS(err, "failed to detect swap behavior; kubelet swapBehavior configuration may be missing or misconfigured") + } else if swapBehavior == "" { + swap["behavior"] = defaultSwapBehavior + } else { + swap["behavior"] = swapBehavior + } } // Detect NVDIMM @@ -155,6 +177,16 @@ func detectSwap() (map[string]string, error) { }, nil } +// detectSwapBehavior detects the swap behavior as configured in the kubelet. +func detectSwapBehavior(configFilePath string) (string, error) { + kubeletConfig, err := kubeconf.ReadKubeletConfig(configFilePath) + if err != nil { + return "", fmt.Errorf("failed to read kubelet configuration file %q: %w", configFilePath, err) + } + + return kubeletConfig.MemorySwap.SwapBehavior, nil +} + // detectNuma detects NUMA node information func detectNuma() (map[string]string, error) { sysfsBasePath := hostpath.SysfsDir.Path("bus/node/devices")