Merge pull request kubernetes#120616 from kannon92/kubelet-disk-api-changes

Kubelet disk api changes

Author: k8s-ci-robot

pkg/features/kube_features.go

@@ -454,6 +454,12 @@ const (
 	// Enable POD resources API to return allocatable resources
 	KubeletPodResourcesGetAllocatable featuregate.Feature = "KubeletPodResourcesGetAllocatable"
 
+	// KubeletSeparateDiskGC enables Kubelet to garbage collection images/containers on different filesystems
+	// owner: @kannon92
+	// kep: https://kep.k8s.io/4191
+	// alpha: v1.29
+	KubeletSeparateDiskGC featuregate.Feature = "KubeletSeparateDiskGC"
+
 	// owner: @sallyom
 	// kep: https://kep.k8s.io/2832
 	// alpha: v1.25
@@ -1088,6 +1094,8 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 
 	KubeletPodResourcesGetAllocatable: {Default: true, PreRelease: featuregate.GA, LockToDefault: true}, // GA in 1.28, remove in 1.30
 
+	KubeletSeparateDiskGC: {Default: false, PreRelease: featuregate.Alpha},
+
 	KubeletTracing: {Default: true, PreRelease: featuregate.Beta},
 
 	KubeProxyDrainingTerminatingNodes: {Default: false, PreRelease: featuregate.Alpha},

pkg/kubelet/cadvisor/cadvisor_linux.go

@@ -186,6 +186,14 @@ func (cc *cadvisorClient) getFsInfo(label string) (cadvisorapiv2.FsInfo, error)
 	return res[0], nil
 }
 
+func (cc *cadvisorClient) ContainerFsInfo() (cadvisorapiv2.FsInfo, error) {
+	label, err := cc.imageFsInfoProvider.ContainerFsInfoLabel()
+	if err != nil {
+		return cadvisorapiv2.FsInfo{}, err
+	}
+	return cc.getFsInfo(label)
+}
+
 func (cc *cadvisorClient) WatchEvents(request *events.Request) (*events.EventChannel, error) {
 	return cc.WatchForEvents(request)
 }

pkg/kubelet/cadvisor/cadvisor_linux_test.go

@@ -57,3 +57,32 @@ func TestImageFsInfoLabel(t *testing.T) {
 		})
 	}
 }
+
+func TestContainerFsInfoLabel(t *testing.T) {
+	testcases := []struct {
+		description     string
+		runtime         string
+		runtimeEndpoint string
+		expectedLabel   string
+		expectedError   error
+	}{{
+		description:     "LabelCrioWriteableImages should be returned",
+		runtimeEndpoint: crio.CrioSocket,
+		expectedLabel:   LabelCrioContainers,
+		expectedError:   nil,
+	}, {
+		description:     "Cannot find valid imagefs label",
+		runtimeEndpoint: "",
+		expectedLabel:   "",
+		expectedError:   fmt.Errorf("no containerfs label for configured runtime"),
+	}}
+
+	for _, tc := range testcases {
+		t.Run(tc.description, func(t *testing.T) {
+			infoProvider := NewImageFsInfoProvider(tc.runtimeEndpoint)
+			label, err := infoProvider.ContainerFsInfoLabel()
+			assert.Equal(t, tc.expectedLabel, label)
+			assert.Equal(t, tc.expectedError, err)
+		})
+	}
+}

pkg/kubelet/cadvisor/cadvisor_unsupported.go

@@ -79,6 +79,10 @@ func (cu *cadvisorUnsupported) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
 	return cadvisorapiv2.FsInfo{}, errUnsupported
 }
 
+func (cu *cadvisorUnsupported) ContainerFsInfo() (cadvisorapiv2.FsInfo, error) {
+	return cadvisorapiv2.FsInfo{}, errUnsupported
+}
+
 func (cu *cadvisorUnsupported) WatchEvents(request *events.Request) (*events.EventChannel, error) {
 	return nil, errUnsupported
 }

pkg/kubelet/cadvisor/cadvisor_windows.go

@@ -79,6 +79,10 @@ func (cu *cadvisorClient) ImagesFsInfo() (cadvisorapiv2.FsInfo, error) {
 	return cadvisorapiv2.FsInfo{}, nil
 }
 
+func (cu *cadvisorClient) ContainerFsInfo() (cadvisorapiv2.FsInfo, error) {
+	return cadvisorapiv2.FsInfo{}, nil
+}
+
 func (cu *cadvisorClient) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
 	return cu.GetDirFsInfo(cu.rootPath)
 }

pkg/kubelet/cadvisor/helpers_linux.go

@@ -26,6 +26,11 @@ import (
 	cadvisorfs "github.com/google/cadvisor/fs"
 )
 
+// LabelCrioContainers is a label to allow for cadvisor to track writeable layers
+// separately from read-only layers.
+// Once CAdvisor upstream changes are merged, we should remove this constant
+const LabelCrioContainers string = "crio-containers"
+
 // imageFsInfoProvider knows how to translate the configured runtime
 // to its file system label for images.
 type imageFsInfoProvider struct {
@@ -35,15 +40,28 @@ type imageFsInfoProvider struct {
 // ImageFsInfoLabel returns the image fs label for the configured runtime.
 // For remote runtimes, it handles additional runtimes natively understood by cAdvisor.
 func (i *imageFsInfoProvider) ImageFsInfoLabel() (string, error) {
-	// This is a temporary workaround to get stats for cri-o from cadvisor
-	// and should be removed.
-	// Related to https://github.com/kubernetes/kubernetes/issues/51798
-	if strings.HasSuffix(i.runtimeEndpoint, CrioSocketSuffix) {
+	if detectCrioWorkaround(i) {
 		return cadvisorfs.LabelCrioImages, nil
 	}
 	return "", fmt.Errorf("no imagefs label for configured runtime")
 }
 
+// ContainerFsInfoLabel returns the container fs label for the configured runtime.
+// For remote runtimes, it handles addition runtimes natively understood by cAdvisor.
+func (i *imageFsInfoProvider) ContainerFsInfoLabel() (string, error) {
+	if detectCrioWorkaround(i) {
+		return LabelCrioContainers, nil
+	}
+	return "", fmt.Errorf("no containerfs label for configured runtime")
+}
+
+// This is a temporary workaround to get stats for cri-o from cadvisor
+// and should be removed.
+// Related to https://github.com/kubernetes/kubernetes/issues/51798
+func detectCrioWorkaround(i *imageFsInfoProvider) bool {
+	return strings.HasSuffix(i.runtimeEndpoint, CrioSocketSuffix)
+}
+
 // NewImageFsInfoProvider returns a provider for the specified runtime configuration.
 func NewImageFsInfoProvider(runtimeEndpoint string) ImageFsInfoProvider {
 	return &imageFsInfoProvider{runtimeEndpoint: runtimeEndpoint}

pkg/kubelet/cadvisor/helpers_unsupported.go

@@ -29,6 +29,10 @@ func (i *unsupportedImageFsInfoProvider) ImageFsInfoLabel() (string, error) {
 	return "", errors.New("unsupported")
 }
 
+func (i *unsupportedImageFsInfoProvider) ContainerFsInfoLabel() (string, error) {
+	return "", errors.New("unsupported")
+}
+
 // NewImageFsInfoProvider returns a provider for the specified runtime configuration.
 func NewImageFsInfoProvider(runtimeEndpoint string) ImageFsInfoProvider {
 	return &unsupportedImageFsInfoProvider{}

pkg/kubelet/cadvisor/testing/cadvisor_fake.go

@@ -101,6 +101,11 @@ func (c *Fake) RootFsInfo() (cadvisorapiv2.FsInfo, error) {
 	return cadvisorapiv2.FsInfo{}, nil
 }
 
+// ContainerFsInfo is a fake implementation of Interface.ContainerFsInfo.
+func (c *Fake) ContainerFsInfo() (cadvisorapiv2.FsInfo, error) {
+	return cadvisorapiv2.FsInfo{}, nil
+}
+
 // WatchEvents is a fake implementation of Interface.WatchEvents.
 func (c *Fake) WatchEvents(request *events.Request) (*events.EventChannel, error) {
 	return new(events.EventChannel), nil

pkg/kubelet/cadvisor/testing/cadvisor_mock.go

@@ -41,6 +41,10 @@ type Interface interface {
 	// Returns usage information about the root filesystem.
 	RootFsInfo() (cadvisorapiv2.FsInfo, error)
 
+	// Returns usage information about the writeable layer.
+	// KEP 4191 can separate the image filesystem
+	ContainerFsInfo() (cadvisorapiv2.FsInfo, error)
+
 	// Get events streamed through passedChannel that fit the request.
 	WatchEvents(request *events.Request) (*events.EventChannel, error)
 
@@ -52,4 +56,6 @@ type Interface interface {
 type ImageFsInfoProvider interface {
 	// ImageFsInfoLabel returns the label cAdvisor should use to find the filesystem holding container images.
 	ImageFsInfoLabel() (string, error)
+	// In split image filesystem this will be different from ImageFsInfoLabel
+	ContainerFsInfoLabel() (string, error)
 }