Discover node features as annotations

Signed-off-by: Carlos Eduardo Arango Gutierrez <[email protected]>
Co-authored-by: bebc <[email protected]>
Co-authored-by: Markus Lehtonen <[email protected]>

Author: 3 people

deployment/base/nfd-crds/nfd-api-crds.yaml

@@ -153,6 +153,11 @@ spec:
                   description: Rule defines a rule for node customization such as
                     labeling.
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      description: Annotations to create if the rule matches.
+                      type: object
                     extendedResources:
                       additionalProperties:
                         type: string

deployment/helm/node-feature-discovery/crds/nfd-api-crds.yaml

@@ -153,6 +153,11 @@ spec:
                   description: Rule defines a rule for node customization such as
                     labeling.
                   properties:
+                    annotations:
+                      additionalProperties:
+                        type: string
+                      description: Annotations to create if the rule matches.
+                      type: object
                     extendedResources:
                       additionalProperties:
                         type: string

docs/get-started/introduction.md

@@ -18,8 +18,8 @@ sort: 1
 This software enables node feature discovery for Kubernetes. It detects
 hardware features available on each node in a Kubernetes cluster, and
 advertises those features using node labels and optionally node extended
-resources and node taints. Node Feature Discovery is compatible with any recent
-version of Kubernetes (v1.21+).
+resources, annotations and node taints. Node Feature Discovery is compatible
+with any recent version of Kubernetes (v1.21+).
 
 NFD consists of four software components:
 

docs/usage/customization-guide.md

@@ -594,6 +594,54 @@ details.
 > labels specified in the `labels` field will override anything
 > originating from `labelsTemplate`.
 
+#### Node Annotations
+
+The `.annotations` field is a list of features to be advertised as annotations.
+
+Take this rule as a referential example:
+
+```yaml
+apiVersion: nfd.k8s-sigs.io/v1alpha1
+kind: NodeFeatureRule
+metadata:
+  name: feature-annotations-example
+spec:
+  rules:
+    - name: "annotation-example"
+      annotations:
+        defaul-ns-annotation: "foo"
+        feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
+        custom.vendor.io/feature: "baz"
+      matchFeatures:
+        - feature: kernel.version
+          matchExpressions:
+            major: {op: Exists}
+```
+
+This will yield into the following node annotations:
+
+```yaml
+  annotations:
+    ...
+    feature.node.kubernetes.io/defaul-ns-annotation: "foo"
+    feature.node.kubernetes.io/defaul-ns-annotation-2: "bar"
+    custom.vendor.io/feature: "baz"
+    ...
+```
+
+NFD enforces some limitations to the namespace (or prefix)/ of the annotations:
+
+- `kubernetes.io/` and its sub-namespaces (like `sub.ns.kubernetes.io/`) cannot
+  generally be used
+- the only exception is `feature.node.kubernetes.io/` and its sub-namespaces
+  (like `sub.ns.feature.node.kubernetes.io`)
+- unprefixed names will get prefixed with `feature.node.kubernetes.io/`
+  automatically (e.g. `foo` becomes `feature.node.kubernetes.io/foo`)
+
+> **NOTE:** The `annotations` field has will only advertise features via node
+> annotations the features won't be advertised as node labels unless they are
+> specified in the `labels` field.
+
 #### Taints
 
 *taints* is a list of taint entries and each entry can have `key`, `value` and `effect`,

pkg/apis/nfd/v1alpha1/annotations_labels.go

@@ -60,9 +60,18 @@ const (
 	// NodeTaintsAnnotation is the annotation that holds the taints that nfd-master set on the node
 	NodeTaintsAnnotation = AnnotationNs + "/taints"
 
+	// FeatureAnnotationsTrackingAnnotation is the annotation that holds all feature annotations that nfd-master set on the node
+	FeatureAnnotationsTrackingAnnotation = AnnotationNs + "/feature-annotations"
+
 	// NodeFeatureObjNodeNameLabel is the label that specifies which node the
 	// NodeFeature object is targeting. Creators of NodeFeature objects must
 	// set this label and consumers of the objects are supposed to use the
 	// label for filtering features designated for a certain node.
 	NodeFeatureObjNodeNameLabel = "nfd.node.kubernetes.io/node-name"
+
+	// FeatureAnnotationNs is the (default) namespace for feature annotations.
+	FeatureAnnotationNs = "feature.node.kubernetes.io"
+
+	// FeatureAnnotationSubNsSuffix is the suffix for allowed feature annotation sub-namespaces.
+	FeatureAnnotationSubNsSuffix = "." + FeatureAnnotationNs
 )

pkg/apis/nfd/v1alpha1/rule.go

@@ -24,6 +24,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog/v2"
+
 	"sigs.k8s.io/node-feature-discovery/pkg/utils"
 )
 
@@ -32,6 +33,7 @@ import (
 type RuleOutput struct {
 	ExtendedResources map[string]string
 	Labels            map[string]string
+	Annotations       map[string]string
 	Vars              map[string]string
 	Taints            []corev1.Taint
 }
@@ -101,7 +103,7 @@ func (r *Rule) Execute(features *Features) (RuleOutput, error) {
 		vars[k] = v
 	}
 
-	ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints}
+	ret := RuleOutput{ExtendedResources: extendedResources, Labels: labels, Vars: vars, Taints: r.Taints, Annotations: r.Annotations}
 	klog.V(2).InfoS("rule matched", "ruleName", r.Name, "ruleOutput", utils.DelayedDumper(ret))
 	return ret, nil
 }

pkg/apis/nfd/v1alpha1/types.go

@@ -146,6 +146,10 @@ type Rule struct {
 	// +optional
 	LabelsTemplate string `json:"labelsTemplate"`
 
+	// Annotations to create if the rule matches.
+	// +optional
+	Annotations map[string]string `json:"annotations"`
+
 	// Vars is the variables to store if the rule matches. Variables do not
 	// directly inflict any changes in the node object. However, they can be
 	// referenced from other rules enabling more complex rule hierarchies,

pkg/apis/nfd/v1alpha1/zz_generated.deepcopy.go

@@ -138,6 +138,12 @@ func TestUpdateNodeObject(t *testing.T) {
 		}
 		sort.Strings(fakeFeatureLabelNames)
 
+		fakeFeatureAnnotationsNames := make([]string, 0, len(fakeFeatureLabels))
+		for k := range fakeAnnotations {
+			fakeFeatureAnnotationsNames = append(fakeFeatureAnnotationsNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureAnnotationNs+"/"))
+		}
+		sort.Strings(fakeFeatureAnnotationsNames)
+
 		fakeExtResourceNames := make([]string, 0, len(fakeExtResources))
 		for k := range fakeExtResources {
 			fakeExtResourceNames = append(fakeExtResourceNames, strings.TrimPrefix(k, nfdv1alpha1.FeatureLabelNs+"/"))
@@ -162,6 +168,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			// Create a list of expected node metadata patches
 			metadataPatches := []apihelper.JsonPatch{
 				apihelper.NewJsonPatch("replace", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/feature-labels", strings.Join(fakeFeatureLabelNames, ",")),
+				apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.FeatureAnnotationsTrackingAnnotation, strings.Join(fakeFeatureAnnotationsNames, ",")),
 				apihelper.NewJsonPatch("add", "/metadata/annotations", nfdv1alpha1.AnnotationNs+"/extended-resources", strings.Join(fakeExtResourceNames, ",")),
 				apihelper.NewJsonPatch("remove", "/metadata/labels", nfdv1alpha1.FeatureLabelNs+"/old-feature", ""),
 			}
@@ -176,7 +183,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice()
 			mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil)
 			mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(metadataPatches))).Return(nil)
-			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is nil", func() {
 				So(err, ShouldBeNil)
@@ -186,7 +193,7 @@ func TestUpdateNodeObject(t *testing.T) {
 		Convey("When I fail to update the node with feature labels", func() {
 			expectedError := fmt.Errorf("no client is passed, client:  <nil>")
 			mockAPIHelper.On("GetClient").Return(nil, expectedError)
-			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err, ShouldResemble, expectedError)
@@ -196,7 +203,7 @@ func TestUpdateNodeObject(t *testing.T) {
 		Convey("When I fail to get a mock client while updating feature labels", func() {
 			expectedError := fmt.Errorf("no client is passed, client:  <nil>")
 			mockAPIHelper.On("GetClient").Return(nil, expectedError)
-			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(nil, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err, ShouldResemble, expectedError)
@@ -207,7 +214,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			expectedError := errors.New("fake error")
 			mockAPIHelper.On("GetClient").Return(mockClient, nil)
 			mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(nil, expectedError).Twice()
-			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err, ShouldEqual, expectedError)
@@ -220,7 +227,7 @@ func TestUpdateNodeObject(t *testing.T) {
 			mockAPIHelper.On("GetNode", mockClient, mockNodeName).Return(mockNode, nil).Twice()
 			mockAPIHelper.On("PatchNodeStatus", mockClient, mockNodeName, mock.MatchedBy(jsonPatchMatcher(statusPatches))).Return(nil)
 			mockAPIHelper.On("PatchNode", mockClient, mockNodeName, mock.Anything).Return(expectedError).Twice()
-			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, fakeAnnotations, fakeExtResources, nil)
+			err := mockMaster.updateNodeObject(mockClient, mockNodeName, fakeFeatureLabels, Annotations{}, fakeAnnotations, fakeExtResources, nil)
 
 			Convey("Error is produced", func() {
 				So(err.Error(), ShouldEndWith, expectedError.Error())