Reduce memory usage/allocations during wait for volume attachment

After a node restart kubelet tries to (re)attach all volumes to the
pods. We poll the `verifyVolumesMountedFunc` every 300ms to check whether
the mount has succeeded. This function called the
`GetMountedVolumesForPod` function that allocates memory for every
volumes on every pod (`len(asw.attachedVolumes)`). Because this function
is executed for every pod simultaneously, this results in exponential
memory usage and high cpu usage due to garbage collection.
We already know the exact volume names and pod name and are able to
completly remove the slice allocation.

Signed-off-by: Luca Berneking <[email protected]>

Author: Lucaber

pkg/kubelet/volumemanager/cache/actual_state_of_world.go

@@ -120,6 +120,9 @@ type ActualStateOfWorld interface {
 	// and false is returned.
 	PodRemovedFromVolume(podName volumetypes.UniquePodName, volumeName v1.UniqueVolumeName) bool
 
+	// PodHasMountedVolumes returns true if any volume is mounted on the given pod
+	PodHasMountedVolumes(podName volumetypes.UniquePodName) bool
+
 	// VolumeExistsWithSpecName returns true if the given volume specified with the
 	// volume spec name (a.k.a., InnerVolumeSpecName) exists in the list of
 	// volumes that should be attached to this node.
@@ -146,6 +149,10 @@ type ActualStateOfWorld interface {
 	// current actual state of the world.
 	GetMountedVolumesForPod(podName volumetypes.UniquePodName) []MountedVolume
 
+	// GetMountedVolumeForPodByOuterVolumeSpecName returns the volume and true if
+	// the given outerVolumeSpecName is mounted on the given pod.
+	GetMountedVolumeForPodByOuterVolumeSpecName(podName volumetypes.UniquePodName, outerVolumeSpecName string) (MountedVolume, bool)
+
 	// GetPossiblyMountedVolumesForPod generates and returns a list of volumes for
 	// the specified pod that either are attached and mounted or are "uncertain",
 	// i.e. a volume plugin may be mounting the volume right now.
@@ -948,6 +955,20 @@ func (asw *actualStateOfWorld) PodExistsInVolume(podName volumetypes.UniquePodNa
 	return podExists, volumeObj.devicePath, nil
 }
 
+func (asw *actualStateOfWorld) PodHasMountedVolumes(podName volumetypes.UniquePodName) bool {
+	asw.RLock()
+	defer asw.RUnlock()
+	for _, volumeObj := range asw.attachedVolumes {
+		if podObj, hasPod := volumeObj.mountedPods[podName]; hasPod {
+			if podObj.volumeMountStateForPod == operationexecutor.VolumeMounted {
+				return true
+			}
+		}
+	}
+
+	return false
+}
+
 func (asw *actualStateOfWorld) volumeNeedsExpansion(volumeObj attachedVolume, desiredVolumeSize resource.Quantity) (resource.Quantity, bool) {
 	currentSize := resource.Quantity{}
 	if volumeObj.persistentVolumeSize != nil {
@@ -1061,7 +1082,7 @@ func (asw *actualStateOfWorld) GetMountedVolumesForPod(
 	podName volumetypes.UniquePodName) []MountedVolume {
 	asw.RLock()
 	defer asw.RUnlock()
-	mountedVolume := make([]MountedVolume, 0 /* len */, len(asw.attachedVolumes) /* cap */)
+	mountedVolume := make([]MountedVolume, 0 /* len */)
 	for _, volumeObj := range asw.attachedVolumes {
 		for mountedPodName, podObj := range volumeObj.mountedPods {
 			if mountedPodName == podName && podObj.volumeMountStateForPod == operationexecutor.VolumeMounted {
@@ -1075,6 +1096,21 @@ func (asw *actualStateOfWorld) GetMountedVolumesForPod(
 	return mountedVolume
 }
 
+func (asw *actualStateOfWorld) GetMountedVolumeForPodByOuterVolumeSpecName(
+	podName volumetypes.UniquePodName, outerVolumeSpecName string) (MountedVolume, bool) {
+	asw.RLock()
+	defer asw.RUnlock()
+	for _, volumeObj := range asw.attachedVolumes {
+		if podObj, hasPod := volumeObj.mountedPods[podName]; hasPod {
+			if podObj.volumeMountStateForPod == operationexecutor.VolumeMounted && podObj.outerVolumeSpecName == outerVolumeSpecName {
+				return getMountedVolume(&podObj, &volumeObj), true
+			}
+		}
+	}
+
+	return MountedVolume{}, false
+}
+
 func (asw *actualStateOfWorld) GetPossiblyMountedVolumesForPod(
 	podName volumetypes.UniquePodName) []MountedVolume {
 	asw.RLock()

pkg/kubelet/volumemanager/volume_manager.go

@@ -514,7 +514,13 @@ func (vm *volumeManager) verifyVolumesMountedFunc(podName types.UniquePodName, e
 		if errs := vm.desiredStateOfWorld.PopPodErrors(podName); len(errs) > 0 {
 			return true, errors.New(strings.Join(errs, "; "))
 		}
-		return len(vm.getUnmountedVolumes(podName, expectedVolumes)) == 0, nil
+		for _, expectedVolume := range expectedVolumes {
+			_, found := vm.actualStateOfWorld.GetMountedVolumeForPodByOuterVolumeSpecName(podName, expectedVolume)
+			if !found {
+				return false, nil
+			}
+		}
+		return true, nil
 	}
 }
 
@@ -525,7 +531,7 @@ func (vm *volumeManager) verifyVolumesUnmountedFunc(podName types.UniquePodName)
 		if errs := vm.desiredStateOfWorld.PopPodErrors(podName); len(errs) > 0 {
 			return true, errors.New(strings.Join(errs, "; "))
 		}
-		return len(vm.actualStateOfWorld.GetMountedVolumesForPod(podName)) == 0, nil
+		return !vm.actualStateOfWorld.PodHasMountedVolumes(podName), nil
 	}
 }