Merge pull request kubernetes#125656 from gyuho/recent-stats-check-error-for-error-level-logging

feat(kubelet/stats): match cadvisor error to lower not found stats log level

Author: k8s-ci-robot

pkg/kubelet/server/stats/summary_sys_containers.go

@@ -20,8 +20,11 @@ limitations under the License.
 package stats
 
 import (
+	"errors"
+
 	"k8s.io/klog/v2"
 
+	cadvisormemory "github.com/google/cadvisor/cache/memory"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	statsapi "k8s.io/kubelet/pkg/apis/stats/v1alpha1"
 	"k8s.io/kubernetes/pkg/kubelet/cm"
@@ -80,7 +83,11 @@ func (sp *summaryProviderImpl) GetSystemContainersCPUAndMemoryStats(nodeConfig c
 		}
 		s, err := sp.provider.GetCgroupCPUAndMemoryStats(cont.name, cont.forceStatsUpdate)
 		if err != nil {
-			klog.ErrorS(err, "Failed to get system container stats", "containerName", cont.name)
+			if errors.Is(err, cadvisormemory.ErrDataNotFound) {
+				klog.V(4).InfoS("cgroup stats not found in memory cache", "containerName", cont.name)
+			} else {
+				klog.ErrorS(err, "Failed to get system container stats", "containerName", cont.name)
+			}
 			continue
 		}
 		s.Name = sys

pkg/kubelet/stats/cadvisor_stats_provider.go

@@ -499,7 +499,7 @@ func getCadvisorContainerInfo(ca cadvisor.Interface) (map[string]cadvisorapiv2.C
 			// response.
 			klog.ErrorS(err, "Partial failure issuing cadvisor.ContainerInfoV2")
 		} else {
-			return nil, fmt.Errorf("failed to get root cgroup stats: %v", err)
+			return nil, fmt.Errorf("failed to get root cgroup stats: %w", err)
 		}
 	}
 	return infos, nil

pkg/kubelet/stats/helper.go

@@ -318,7 +318,7 @@ func getCgroupInfo(cadvisor cadvisor.Interface, containerName string, updateStat
 		MaxAge:    maxAge,
 	})
 	if err != nil {
-		return nil, fmt.Errorf("failed to get container info for %q: %v", containerName, err)
+		return nil, fmt.Errorf("failed to get container info for %q: %w", containerName, err)
 	}
 	if len(infoMap) != 1 {
 		return nil, fmt.Errorf("unexpected number of containers: %v", len(infoMap))

pkg/kubelet/stats/provider.go

@@ -18,8 +18,10 @@ package stats
 
 import (
 	"context"
+	"errors"
 	"fmt"
 
+	cadvisormemory "github.com/google/cadvisor/cache/memory"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
 	internalapi "k8s.io/cri-api/pkg/apis"
@@ -113,6 +115,9 @@ func (p *Provider) RlimitStats() (*statsapi.RlimitStats, error) {
 func (p *Provider) GetCgroupStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, *statsapi.NetworkStats, error) {
 	info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats)
 	if err != nil {
+		if errors.Is(err, cadvisormemory.ErrDataNotFound) {
+			return nil, nil, fmt.Errorf("cgroup stats not found for %q: %w", cgroupName, cadvisormemory.ErrDataNotFound)
+		}
 		return nil, nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
 	}
 	// Rootfs and imagefs doesn't make sense for raw cgroup.
@@ -126,6 +131,9 @@ func (p *Provider) GetCgroupStats(cgroupName string, updateStats bool) (*statsap
 func (p *Provider) GetCgroupCPUAndMemoryStats(cgroupName string, updateStats bool) (*statsapi.ContainerStats, error) {
 	info, err := getCgroupInfo(p.cadvisor, cgroupName, updateStats)
 	if err != nil {
+		if errors.Is(err, cadvisormemory.ErrDataNotFound) {
+			return nil, fmt.Errorf("cgroup stats not found for %q: %w", cgroupName, cadvisormemory.ErrDataNotFound)
+		}
 		return nil, fmt.Errorf("failed to get cgroup stats for %q: %v", cgroupName, err)
 	}
 	// Rootfs and imagefs doesn't make sense for raw cgroup.