feat(nodeadm): pass nvidia gpu startup labels to kubelet (#2607)

* feat(nodeadm): pass nvidia gpu startup labels to kubelet

* docs: add feature gate and version-specific behavior table

* tweak testing

e2e test is failing in CI and passing locally. Replace it with a test
using a faked file system.

---------

Co-authored-by: Nick Baker <nbakerd@amazon.com>

Author: fletcherw

nodeadm/doc/api-concepts.md

@@ -18,4 +18,18 @@ There are three levels of stability and support:
 
 ### Stable
 - Example: `v5`.
-- Support for a stable API will align with the support of a major version of Amazon Linux.
+- Support for a stable API will align with the support of a major version of Amazon Linux.
+
+## Feature Gates
+
+|Name|Default|Since|Until|
+|---|---|---|---|
+|`InstanceIdNodeName`|`false`|-|-|
+
+## Behaviorial Boundaries
+
+|Description|Since|Until|
+|---|---|---|
+|Apply the `nvidia.com/gpu.present=true` node label on startup if NVIDIA devices are detected on the instance|1.35|-|
+|Enable CDI in the default containerd configuration|1.32|-|
+|Write user-provided kubelet config as a kubelet drop-in configuration file|1.29|-|

nodeadm/internal/kubelet/config.go

@@ -213,6 +213,26 @@ func (ksc *kubeletConfig) withOutpostSetup(cfg *api.NodeConfig) error {
 	return nil
 }
 
+func (ksc *kubeletConfig) withNodeLabels(flags map[string]string, nodeLabelFuncs map[string]LabelProvider) {
+	var nodeLabels []string
+	for nodeLabelKey, provider := range nodeLabelFuncs {
+		nodeLabelValue, ok, err := provider.Get()
+		if err != nil {
+			zap.L().Error("Failed to get node label value", zap.String("key", nodeLabelKey), zap.Error(err))
+			continue
+		}
+		if !ok {
+			continue
+		}
+		nodeLabel := fmt.Sprintf("%s=%s", nodeLabelKey, nodeLabelValue)
+		zap.L().Info("Adding node label", zap.String("label", nodeLabel))
+		nodeLabels = append(nodeLabels, nodeLabel)
+	}
+	if len(nodeLabels) > 0 {
+		flags["node-labels"] = strings.Join(nodeLabels, ",")
+	}
+}
+
 func (ksc *kubeletConfig) withNodeIp(cfg *api.NodeConfig, flags map[string]string) error {
 	nodeIp, err := getNodeIp(context.TODO(), cfg, imds.DefaultClient())
 	if err != nil {
@@ -321,6 +341,13 @@ func (k *kubelet) GenerateKubeletConfig(cfg *api.NodeConfig) (*kubeletConfig, er
 	kubeletConfig.withDefaultReservedResources(cfg, k.resources)
 	kubeletConfig.withImageServiceEndpoint(cfg, k.resources)
 
+	nodeLabelFuncs := map[string]LabelProvider{}
+	if semver.Compare(cfg.Status.KubeletVersion, "v1.35.0") >= 0 {
+		// see: https://github.com/NVIDIA/gpu-operator/commit/e25291b86cf4542ac62d8635cda4bd653c4face3
+		nodeLabelFuncs["nvidia.com/gpu.present"] = NvidiaGPULabel{fs: system.RealFileSystem{}}
+	}
+	kubeletConfig.withNodeLabels(k.flags, nodeLabelFuncs)
+
 	return &kubeletConfig, nil
 }
 

nodeadm/internal/kubelet/labels.go

@@ -0,0 +1,24 @@
+package kubelet
+
+import (
+	"github.com/awslabs/amazon-eks-ami/nodeadm/internal/system"
+)
+
+type LabelProvider interface {
+	Get() (string, bool, error)
+}
+
+type NvidiaGPULabel struct {
+	fs system.FileSystem
+}
+
+func (n NvidiaGPULabel) Get() (string, bool, error) {
+	ok, err := system.IsPCIVendorAttached(n.fs, system.NVIDIA_VENDOR_ID)
+	if err != nil {
+		return "", false, err
+	}
+	if !ok {
+		return "", false, nil
+	}
+	return "true", true, nil
+}

nodeadm/internal/kubelet/labels_test.go

@@ -0,0 +1,52 @@
+package kubelet
+
+import (
+	"testing"
+
+	"github.com/awslabs/amazon-eks-ami/nodeadm/internal/system"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestNvidiaGPULabel(t *testing.T) {
+	tests := []struct {
+		name          string
+		files         map[string]string
+		expectedValue string
+		expectedOk    bool
+	}{
+		{
+			name: "nvidia gpu present",
+			files: map[string]string{
+				"/sys/bus/pci/devices/0000:00:1e.0/vendor": "0x10de",
+			},
+			expectedValue: "true",
+			expectedOk:    true,
+		},
+		{
+			name: "no nvidia gpu",
+			files: map[string]string{
+				"/sys/bus/pci/devices/0000:00:1e.0/vendor": "0x1234",
+			},
+			expectedValue: "",
+			expectedOk:    false,
+		},
+		{
+			name: "no files at all",
+			files: map[string]string{
+				"/sys/bus/pci/devices/": system.EmptyDirectoryMarker,
+			},
+			expectedValue: "",
+			expectedOk:    false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			label := NvidiaGPULabel{fs: system.FakeFileSystem{Files: tt.files}}
+			value, ok, err := label.Get()
+			assert.NoError(t, err)
+			assert.Equal(t, tt.expectedValue, value)
+			assert.Equal(t, tt.expectedOk, ok)
+		})
+	}
+}

nodeadm/internal/system/devices.go

@@ -0,0 +1,30 @@
+package system
+
+import (
+	"strings"
+
+	"go.uber.org/zap"
+)
+
+const NVIDIA_VENDOR_ID = "0x10de"
+
+// IsPCIVendorAttached returns whether any pcie devices with a given vendor id
+// are attached to the instance.
+func IsPCIVendorAttached(fs FileSystem, vendorId string) (bool, error) {
+	vendorPaths, err := fs.Glob("/sys/bus/pci/devices/*/vendor")
+	if err != nil {
+		return false, err
+	}
+	for _, vendorPath := range vendorPaths {
+		// #nosec G304 // read only operation on sysfs path
+		vendorIdBytes, err := fs.ReadFile(vendorPath)
+		if err != nil {
+			zap.L().Warn("failed to read vendor id", zap.Error(err))
+			continue
+		}
+		if strings.TrimSpace(string(vendorIdBytes)) == vendorId {
+			return true, nil
+		}
+	}
+	return false, nil
+}