nfd-worker: Watch features.d changes

Signed-off-by: Oleg Zhurakivskyy <[email protected]>

Author: ozhuraki

pkg/nfd-worker/nfd-worker-internal_test.go

@@ -23,6 +23,8 @@ import (
 	"testing"
 	"time"
 
+	"golang.org/x/net/context"
+
 	. "github.com/smartystreets/goconvey/convey"
 	"github.com/vektra/errors"
 	fakeclient "k8s.io/client-go/kubernetes/fake"
@@ -102,7 +104,7 @@ func TestConfigParse(t *testing.T) {
 		overrides := `{"core": {"labelSources": ["fake"],"noPublish": true},"sources": {"cpu": {"cpuid": {"attributeBlacklist": ["foo","bar"]}}}}`
 
 		Convey("and no core cmdline flags have been specified", func() {
-			So(worker.configure("non-existing-file", overrides), ShouldBeNil)
+			So(worker.configure(context.Background(), "non-existing-file", overrides), ShouldBeNil)
 
 			Convey("core overrides should be in effect", func() {
 				So(worker.config.Core.LabelSources, ShouldResemble, []string{"fake"})
@@ -114,7 +116,7 @@ func TestConfigParse(t *testing.T) {
 			worker.args = Args{Overrides: ConfigOverrideArgs{
 				LabelSources:   &utils.StringSliceVal{"cpu", "kernel", "pci"},
 				FeatureSources: &utils.StringSliceVal{"cpu"}}}
-			So(worker.configure("non-existing-file", overrides), ShouldBeNil)
+			So(worker.configure(context.Background(), "non-existing-file", overrides), ShouldBeNil)
 
 			Convey("core cmdline flags should be in effect instead overrides", func() {
 				So(worker.config.Core.LabelSources, ShouldResemble, []string{"cpu", "kernel", "pci"})
@@ -150,7 +152,7 @@ sources:
 
 		Convey("and a proper config file is specified", func() {
 			worker.args = Args{Overrides: ConfigOverrideArgs{LabelSources: &utils.StringSliceVal{"cpu", "kernel", "pci"}}}
-			So(worker.configure(f.Name(), ""), ShouldBeNil)
+			So(worker.configure(context.Background(), f.Name(), ""), ShouldBeNil)
 
 			Convey("specified configuration should take effect", func() {
 				// Verify core config
@@ -172,7 +174,7 @@ sources:
 		Convey("and a proper config file and overrides are given", func() {
 			worker.args = Args{Overrides: ConfigOverrideArgs{FeatureSources: &utils.StringSliceVal{"cpu"}}}
 			overrides := `{"core": {"labelSources": ["fake"],"noPublish": true},"sources": {"pci": {"deviceClassWhitelist": ["03"]}}}`
-			So(worker.configure(f.Name(), overrides), ShouldBeNil)
+			So(worker.configure(context.Background(), f.Name(), overrides), ShouldBeNil)
 
 			Convey("overrides should take precedence over the config file", func() {
 				// Verify core config
@@ -205,7 +207,7 @@ func TestNewNfdWorker(t *testing.T) {
 				So(err, ShouldBeNil)
 			})
 			worker := w.(*nfdWorker)
-			So(worker.configure("", ""), ShouldBeNil)
+			So(worker.configure(context.Background(), "", ""), ShouldBeNil)
 			Convey("all sources should be enabled and the whitelist regexp should be empty", func() {
 				So(len(worker.featureSources), ShouldEqual, len(source.GetAllFeatureSources())-1)
 				So(len(worker.labelSources), ShouldEqual, len(source.GetAllLabelSources())-1)
@@ -223,7 +225,7 @@ func TestNewNfdWorker(t *testing.T) {
 				So(err, ShouldBeNil)
 			})
 			worker := w.(*nfdWorker)
-			So(worker.configure("", ""), ShouldBeNil)
+			So(worker.configure(context.Background(), "", ""), ShouldBeNil)
 			Convey("proper sources should be enabled", func() {
 				So(len(worker.featureSources), ShouldEqual, 1)
 				So(worker.featureSources[0].Name(), ShouldEqual, "cpu")

pkg/nfd-worker/nfd-worker.go

@@ -120,7 +120,8 @@ type nfdWorker struct {
 	kubernetesNamespace string
 	k8sClient           k8sclient.Interface
 	nfdClient           nfdclient.Interface
-	stop                chan struct{} // channel for signaling stop
+	stop                chan struct{}              // channel for signaling stop
+	sourceEvent         chan *source.FeatureSource // channel for events from sources
 	featureSources      []source.FeatureSource
 	labelSources        []source.LabelSource
 	ownerReference      []metav1.OwnerReference
@@ -220,6 +221,19 @@ func (i *infiniteTicker) Reset(d time.Duration) {
 	}
 }
 
+// Publish labels.
+func (w *nfdWorker) publishLabels() error {
+	// Get the set of feature labels.
+	labels := createFeatureLabels(w.labelSources, w.config.Core.LabelWhiteList.Regexp)
+
+	// Update the node with the feature labels.
+	if !w.config.Core.NoPublish {
+		return w.advertiseFeatures(labels)
+	}
+
+	return nil
+}
+
 // Run feature discovery.
 func (w *nfdWorker) runFeatureDiscovery() error {
 	discoveryStart := time.Now()
@@ -237,12 +251,9 @@ func (w *nfdWorker) runFeatureDiscovery() error {
 	if w.config.Core.SleepInterval.Duration > 0 && discoveryDuration > w.config.Core.SleepInterval.Duration/2 {
 		klog.InfoS("feature discovery sources took over half of sleep interval ", "duration", discoveryDuration, "sleepInterval", w.config.Core.SleepInterval.Duration)
 	}
-	// Get the set of feature labels.
-	labels := createFeatureLabels(w.labelSources, w.config.Core.LabelWhiteList.Regexp)
 
-	// Update the node with the feature labels.
-	if !w.config.Core.NoPublish {
-		return w.advertiseFeatures(labels)
+	if err := w.publishLabels(); err != nil {
+		return err
 	}
 
 	return nil
@@ -293,8 +304,11 @@ func (w *nfdWorker) setOwnerReference() error {
 func (w *nfdWorker) Run() error {
 	klog.InfoS("Node Feature Discovery Worker", "version", version.Get(), "nodeName", utils.NodeName(), "namespace", w.kubernetesNamespace)
 
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+
 	// Read configuration file
-	err := w.configure(w.configFilePath, w.args.Options)
+	err := w.configure(ctx, w.configFilePath, w.args.Options)
 	if err != nil {
 		return err
 	}
@@ -341,6 +355,15 @@ func (w *nfdWorker) Run() error {
 				return err
 			}
 
+		case s := <-w.sourceEvent:
+			if err := (*s).Discover(); err != nil {
+				klog.ErrorS(err, "feature discovery failed", "source", (*s).Name())
+				break
+			}
+			if err = w.publishLabels(); err != nil {
+				return err
+			}
+
 		case <-w.stop:
 			klog.InfoS("shutting down nfd-worker")
 			return nil
@@ -461,7 +484,7 @@ func (w *nfdWorker) configureCore(c coreConfig) error {
 }
 
 // Parse configuration options
-func (w *nfdWorker) configure(filepath string, overrides string) error {
+func (w *nfdWorker) configure(ctx context.Context, filepath string, overrides string) error {
 	// Create a new default config
 	c := newDefaultConfig()
 	confSources := source.GetAllConfigurableSources()
@@ -525,6 +548,14 @@ func (w *nfdWorker) configure(filepath string, overrides string) error {
 		s.SetConfig(c.Sources[s.Name()])
 	}
 
+	w.sourceEvent = make(chan *source.FeatureSource)
+	eventSources := source.GetAllEventSources()
+	for _, s := range eventSources {
+		if err := s.SetNotifyChannel(ctx, w.sourceEvent); err != nil {
+			klog.ErrorS(err, "failed to set notify channel for event source", "source", s.Name())
+		}
+	}
+
 	klog.InfoS("configuration successfully updated", "configuration", w.config)
 	return nil
 }

source/local/local.go

@@ -24,8 +24,10 @@ import (
 	"strings"
 	"time"
 
+	"golang.org/x/net/context"
 	"k8s.io/klog/v2"
 
+	"github.com/fsnotify/fsnotify"
 	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1"
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
 	"sigs.k8s.io/node-feature-discovery/source"
@@ -65,10 +67,11 @@ var (
 	featureFilesDir = "/etc/kubernetes/node-feature-discovery/features.d/"
 )
 
-// localSource implements the FeatureSource and LabelSource interfaces.
+// localSource implements the FeatureSource, LabelSource, EventSource interfaces.
 type localSource struct {
-	features *nfdv1alpha1.Features
-	config   *Config
+	features  *nfdv1alpha1.Features
+	config    *Config
+	fsWatcher *fsnotify.Watcher
 }
 
 type Config struct {
@@ -87,6 +90,7 @@ var (
 	_   source.FeatureSource      = &src
 	_   source.LabelSource        = &src
 	_   source.ConfigurableSource = &src
+	_   source.EventSource        = &src
 )
 
 // Name method of the LabelSource interface
@@ -318,6 +322,63 @@ func getFileContent(fileName string) ([][]byte, error) {
 	return lines, nil
 }
 
+func (s *localSource) runNotifier(ctx context.Context, ch chan *source.FeatureSource) {
+	rateLimit := time.After(time.Second)
+	limit := false
+	for {
+		select {
+		case event := <-s.fsWatcher.Events:
+			opAny := fsnotify.Create | fsnotify.Write | fsnotify.Remove | fsnotify.Rename | fsnotify.Chmod
+			if event.Op&opAny != 0 {
+				klog.V(2).InfoS("fsnotify event", "eventName", event.Name, "eventOp", event.Op)
+				if !limit {
+					fs := source.FeatureSource(s)
+					ch <- &fs
+					limit = true
+				}
+			}
+		case err := <-s.fsWatcher.Errors:
+			klog.ErrorS(err, "failed to watch features.d changes")
+		case <-rateLimit:
+			rateLimit = time.After(time.Second)
+			limit = false
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// SetNotifyChannel method of the EventSource Interface
+func (s *localSource) SetNotifyChannel(ctx context.Context, ch chan *source.FeatureSource) error {
+	info, err := os.Stat(featureFilesDir)
+	if err != nil {
+		if !os.IsNotExist(err) {
+			return err
+		}
+	}
+
+	if info != nil && info.IsDir() {
+		watcher, err := fsnotify.NewWatcher()
+		if err != nil {
+			return err
+		}
+
+		err = watcher.Add(featureFilesDir)
+		if err != nil {
+			errWatcher := watcher.Close()
+			if errWatcher != nil {
+				klog.ErrorS(errWatcher, "failed to close fsnotify watcher")
+			}
+			return fmt.Errorf("unable to access %v: %w", featureFilesDir, err)
+		}
+		s.fsWatcher = watcher
+
+		go s.runNotifier(ctx, ch)
+	}
+
+	return nil
+}
+
 func init() {
 	source.Register(&src)
 }

source/source.go

@@ -21,6 +21,8 @@ package source
 import (
 	"fmt"
 
+	"golang.org/x/net/context"
+
 	nfdv1alpha1 "sigs.k8s.io/node-feature-discovery/api/nfd/v1alpha1"
 )
 
@@ -77,6 +79,14 @@ type SupplementalSource interface {
 	DisableByDefault() bool
 }
 
+// EventSource is an interface for a source that can send events
+type EventSource interface {
+	FeatureSource
+
+	// SetNotifyChannel sets the notification channel used to send updates about feature changes.
+	SetNotifyChannel(ctx context.Context, ch chan *FeatureSource) error
+}
+
 // FeatureLabelValue represents the value of one feature label
 type FeatureLabelValue interface{}
 
@@ -155,6 +165,17 @@ func GetAllConfigurableSources() map[string]ConfigurableSource {
 	return all
 }
 
+// GetAllEventSources returns all registered event sources
+func GetAllEventSources() map[string]EventSource {
+	all := make(map[string]EventSource)
+	for k, v := range sources {
+		if s, ok := v.(EventSource); ok {
+			all[k] = s
+		}
+	}
+	return all
+}
+
 // GetAllFeatures returns a combined set of all features from all feature
 // sources.
 func GetAllFeatures() *nfdv1alpha1.Features {