Skip to content

Commit 68451f3

Browse files
authored
Merge pull request kubernetes#72291 from msau42/fix-subpath-orphan
Fix subpath issues with orphaned pod cleanup
2 parents 915c81f + 3ebbbbd commit 68451f3

File tree

6 files changed

+195
-2
lines changed

6 files changed

+195
-2
lines changed

pkg/kubelet/BUILD

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -165,6 +165,7 @@ go_test(
165165
"kubelet_pods_windows_test.go",
166166
"kubelet_resources_test.go",
167167
"kubelet_test.go",
168+
"kubelet_volumes_linux_test.go",
168169
"kubelet_volumes_test.go",
169170
"oom_watcher_test.go",
170171
"pod_container_deletor_test.go",

pkg/kubelet/config/defaults.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package config
1919
const (
2020
DefaultKubeletPodsDirName = "pods"
2121
DefaultKubeletVolumesDirName = "volumes"
22+
DefaultKubeletVolumeSubpathsDirName = "volume-subpaths"
2223
DefaultKubeletVolumeDevicesDirName = "volumeDevices"
2324
DefaultKubeletPluginsDirName = "plugins"
2425
DefaultKubeletPluginsRegistrationDirName = "plugins_registry"

pkg/kubelet/kubelet_getters.go

Lines changed: 21 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ import (
2525
cadvisorapiv1 "github.com/google/cadvisor/info/v1"
2626
"k8s.io/klog"
2727

28-
"k8s.io/api/core/v1"
28+
v1 "k8s.io/api/core/v1"
2929
"k8s.io/apimachinery/pkg/types"
3030
"k8s.io/kubernetes/pkg/kubelet/cm"
3131
"k8s.io/kubernetes/pkg/kubelet/config"
@@ -99,6 +99,13 @@ func (kl *Kubelet) getPodDir(podUID types.UID) string {
9999
return filepath.Join(kl.getPodsDir(), string(podUID))
100100
}
101101

102+
// getPodVolumesSubpathsDir returns the full path to the per-pod subpaths directory under
103+
// which subpath volumes are created for the specified pod. This directory may not
104+
// exist if the pod does not exist or subpaths are not specified.
105+
func (kl *Kubelet) getPodVolumeSubpathsDir(podUID types.UID) string {
106+
return filepath.Join(kl.getPodDir(podUID), config.DefaultKubeletVolumeSubpathsDirName)
107+
}
108+
102109
// getPodVolumesDir returns the full path to the per-pod data directory under
103110
// which volumes are created for the specified pod. This directory may not
104111
// exist if the pod does not exist.
@@ -315,6 +322,19 @@ func (kl *Kubelet) getMountedVolumePathListFromDisk(podUID types.UID) ([]string,
315322
return mountedVolumes, nil
316323
}
317324

325+
// podVolumesSubpathsDirExists returns true if the pod volume-subpaths directory for
326+
// a given pod exists
327+
func (kl *Kubelet) podVolumeSubpathsDirExists(podUID types.UID) (bool, error) {
328+
podVolDir := kl.getPodVolumeSubpathsDir(podUID)
329+
330+
if pathExists, pathErr := volumeutil.PathExists(podVolDir); pathErr != nil {
331+
return true, fmt.Errorf("Error checking if path %q exists: %v", podVolDir, pathErr)
332+
} else if !pathExists {
333+
return false, nil
334+
}
335+
return true, nil
336+
}
337+
318338
// GetVersionInfo returns information about the version of cAdvisor in use.
319339
func (kl *Kubelet) GetVersionInfo() (*cadvisorapiv1.VersionInfo, error) {
320340
return kl.cadvisor.VersionInfo()

pkg/kubelet/kubelet_volumes.go

Lines changed: 15 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ package kubelet
1919
import (
2020
"fmt"
2121

22-
"k8s.io/api/core/v1"
22+
v1 "k8s.io/api/core/v1"
2323
"k8s.io/apimachinery/pkg/types"
2424
utilerrors "k8s.io/apimachinery/pkg/util/errors"
2525
"k8s.io/apimachinery/pkg/util/sets"
@@ -114,6 +114,8 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecon
114114
}
115115
// If volumes have not been unmounted/detached, do not delete directory.
116116
// Doing so may result in corruption of data.
117+
// TODO: getMountedVolumePathListFromDisk() call may be redundant with
118+
// kl.getPodVolumePathListFromDisk(). Can this be cleaned up?
117119
if podVolumesExist := kl.podVolumesExist(uid); podVolumesExist {
118120
klog.V(3).Infof("Orphaned pod %q found, but volumes are not cleaned up", uid)
119121
continue
@@ -128,6 +130,18 @@ func (kl *Kubelet) cleanupOrphanedPodDirs(pods []*v1.Pod, runningPods []*kubecon
128130
orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume paths are still present on disk", uid))
129131
continue
130132
}
133+
134+
// If there are any volume-subpaths, do not cleanup directories
135+
volumeSubpathExists, err := kl.podVolumeSubpathsDirExists(uid)
136+
if err != nil {
137+
orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but error %v occurred during reading of volume-subpaths dir from disk", uid, err))
138+
continue
139+
}
140+
if volumeSubpathExists {
141+
orphanVolumeErrors = append(orphanVolumeErrors, fmt.Errorf("Orphaned pod %q found, but volume subpaths are still present on disk", uid))
142+
continue
143+
}
144+
131145
klog.V(3).Infof("Orphaned pod %q found, removing", uid)
132146
if err := removeall.RemoveAllOneFilesystem(kl.mounter, kl.getPodDir(uid)); err != nil {
133147
klog.Errorf("Failed to remove orphaned pod %q dir; err: %v", uid, err)
Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// +build linux
2+
3+
/*
4+
Copyright 2018 The Kubernetes Authors.
5+
6+
Licensed under the Apache License, Version 2.0 (the "License");
7+
you may not use this file except in compliance with the License.
8+
You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
*/
18+
19+
package kubelet
20+
21+
import (
22+
"fmt"
23+
"io/ioutil"
24+
"os"
25+
"path/filepath"
26+
"testing"
27+
28+
v1 "k8s.io/api/core/v1"
29+
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
30+
31+
_ "k8s.io/kubernetes/pkg/apis/core/install"
32+
)
33+
34+
func validateDirExists(dir string) error {
35+
_, err := ioutil.ReadDir(dir)
36+
if err != nil {
37+
return err
38+
}
39+
return nil
40+
}
41+
42+
func validateDirNotExists(dir string) error {
43+
_, err := ioutil.ReadDir(dir)
44+
if os.IsNotExist(err) {
45+
return nil
46+
}
47+
if err != nil {
48+
return err
49+
}
50+
return fmt.Errorf("dir %q still exists", dir)
51+
}
52+
53+
func TestCleanupOrphanedPodDirs(t *testing.T) {
54+
testCases := map[string]struct {
55+
pods []*v1.Pod
56+
prepareFunc func(kubelet *Kubelet) error
57+
validateFunc func(kubelet *Kubelet) error
58+
expectErr bool
59+
}{
60+
"nothing-to-do": {},
61+
"pods-dir-not-found": {
62+
prepareFunc: func(kubelet *Kubelet) error {
63+
return os.Remove(kubelet.getPodsDir())
64+
},
65+
expectErr: true,
66+
},
67+
"pod-doesnot-exist-novolume": {
68+
prepareFunc: func(kubelet *Kubelet) error {
69+
podDir := kubelet.getPodDir("pod1uid")
70+
return os.MkdirAll(filepath.Join(podDir, "not/a/volume"), 0750)
71+
},
72+
validateFunc: func(kubelet *Kubelet) error {
73+
podDir := kubelet.getPodDir("pod1uid")
74+
return validateDirNotExists(filepath.Join(podDir, "not"))
75+
},
76+
},
77+
"pod-exists-with-volume": {
78+
pods: []*v1.Pod{
79+
{
80+
ObjectMeta: metav1.ObjectMeta{
81+
Name: "pod1",
82+
UID: "pod1uid",
83+
},
84+
},
85+
},
86+
prepareFunc: func(kubelet *Kubelet) error {
87+
podDir := kubelet.getPodDir("pod1uid")
88+
return os.MkdirAll(filepath.Join(podDir, "volumes/plugin/name"), 0750)
89+
},
90+
validateFunc: func(kubelet *Kubelet) error {
91+
podDir := kubelet.getPodDir("pod1uid")
92+
return validateDirExists(filepath.Join(podDir, "volumes/plugin/name"))
93+
},
94+
},
95+
"pod-doesnot-exist-with-volume": {
96+
prepareFunc: func(kubelet *Kubelet) error {
97+
podDir := kubelet.getPodDir("pod1uid")
98+
return os.MkdirAll(filepath.Join(podDir, "volumes/plugin/name"), 0750)
99+
},
100+
validateFunc: func(kubelet *Kubelet) error {
101+
podDir := kubelet.getPodDir("pod1uid")
102+
return validateDirExists(filepath.Join(podDir, "volumes/plugin/name"))
103+
},
104+
},
105+
"pod-doesnot-exist-with-subpath": {
106+
prepareFunc: func(kubelet *Kubelet) error {
107+
podDir := kubelet.getPodDir("pod1uid")
108+
return os.MkdirAll(filepath.Join(podDir, "volume-subpaths/volume/container/index"), 0750)
109+
},
110+
validateFunc: func(kubelet *Kubelet) error {
111+
podDir := kubelet.getPodDir("pod1uid")
112+
return validateDirExists(filepath.Join(podDir, "volume-subpaths/volume/container/index"))
113+
},
114+
},
115+
"pod-doesnot-exist-with-subpath-top": {
116+
prepareFunc: func(kubelet *Kubelet) error {
117+
podDir := kubelet.getPodDir("pod1uid")
118+
return os.MkdirAll(filepath.Join(podDir, "volume-subpaths"), 0750)
119+
},
120+
validateFunc: func(kubelet *Kubelet) error {
121+
podDir := kubelet.getPodDir("pod1uid")
122+
return validateDirExists(filepath.Join(podDir, "volume-subpaths"))
123+
},
124+
},
125+
// TODO: test volume in volume-manager
126+
}
127+
128+
for name, tc := range testCases {
129+
t.Run(name, func(t *testing.T) {
130+
testKubelet := newTestKubelet(t, false /* controllerAttachDetachEnabled */)
131+
defer testKubelet.Cleanup()
132+
kubelet := testKubelet.kubelet
133+
134+
if tc.prepareFunc != nil {
135+
if err := tc.prepareFunc(kubelet); err != nil {
136+
t.Fatalf("%s failed preparation: %v", name, err)
137+
}
138+
}
139+
140+
err := kubelet.cleanupOrphanedPodDirs(tc.pods, nil)
141+
if tc.expectErr && err == nil {
142+
t.Errorf("%s failed: expected error, got success", name)
143+
}
144+
if !tc.expectErr && err != nil {
145+
t.Errorf("%s failed: got error %v", name, err)
146+
}
147+
148+
if tc.validateFunc != nil {
149+
if err := tc.validateFunc(kubelet); err != nil {
150+
t.Errorf("%s failed validation: %v", name, err)
151+
}
152+
}
153+
154+
})
155+
}
156+
}

pkg/util/mount/mount_linux.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ const (
5555
fsckErrorsUncorrected = 4
5656

5757
// place for subpath mounts
58+
// TODO: pass in directory using kubelet_getters instead
5859
containerSubPathDirectoryName = "volume-subpaths"
5960
// syscall.Openat flags used to traverse directories not following symlinks
6061
nofollowFlags = unix.O_RDONLY | unix.O_NOFOLLOW

0 commit comments

Comments
 (0)