Skip to content

Commit 372ebd2

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request kubernetes#83098 from ddebroy/disable-intree
CSI Migration phase 2: disable probing of in-tree plugins
2 parents cb2684c + 129f153 commit 372ebd2

File tree

75 files changed

+1218
-849
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

75 files changed

+1218
-849
lines changed

‎cmd/kube-controller-manager/app/BUILD‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -92,6 +92,7 @@ go_library(
9292
"//pkg/volume/azure_file:go_default_library",
9393
"//pkg/volume/cinder:go_default_library",
9494
"//pkg/volume/csi:go_default_library",
95+
"//pkg/volume/csimigration:go_default_library",
9596
"//pkg/volume/fc:go_default_library",
9697
"//pkg/volume/flexvolume:go_default_library",
9798
"//pkg/volume/flocker:go_default_library",
@@ -140,10 +141,12 @@ go_library(
140141
"//staging/src/k8s.io/cloud-provider:go_default_library",
141142
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
142143
"//staging/src/k8s.io/component-base/cli/globalflag:go_default_library",
144+
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
143145
"//staging/src/k8s.io/component-base/metrics/prometheus/ratelimiter:go_default_library",
144146
"//staging/src/k8s.io/component-base/version:go_default_library",
145147
"//staging/src/k8s.io/component-base/version/verflag:go_default_library",
146148
"//staging/src/k8s.io/csi-translation-lib:go_default_library",
149+
"//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library",
147150
"//staging/src/k8s.io/metrics/pkg/client/clientset/versioned/typed/metrics/v1beta1:go_default_library",
148151
"//staging/src/k8s.io/metrics/pkg/client/custom_metrics:go_default_library",
149152
"//staging/src/k8s.io/metrics/pkg/client/external_metrics:go_default_library",

‎cmd/kube-controller-manager/app/core.go‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ import (
6565
kubefeatures "k8s.io/kubernetes/pkg/features"
6666
"k8s.io/kubernetes/pkg/quota/v1/generic"
6767
quotainstall "k8s.io/kubernetes/pkg/quota/v1/install"
68+
"k8s.io/kubernetes/pkg/volume/csimigration"
6869
netutils "k8s.io/utils/net"
6970
)
7071

@@ -254,10 +255,14 @@ func startRouteController(ctx ControllerContext) (http.Handler, bool, error) {
254255
}
255256

256257
func startPersistentVolumeBinderController(ctx ControllerContext) (http.Handler, bool, error) {
258+
plugins, err := ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration)
259+
if err != nil {
260+
return nil, true, fmt.Errorf("failed to probe volume plugins when starting persistentvolume controller: %v", err)
261+
}
257262
params := persistentvolumecontroller.ControllerParameters{
258263
KubeClient: ctx.ClientBuilder.ClientOrDie("persistent-volume-binder"),
259264
SyncPeriod: ctx.ComponentConfig.PersistentVolumeBinderController.PVClaimBinderSyncPeriod.Duration,
260-
VolumePlugins: ProbeControllerVolumePlugins(ctx.Cloud, ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
265+
VolumePlugins: plugins,
261266
Cloud: ctx.Cloud,
262267
ClusterName: ctx.ComponentConfig.KubeCloudShared.ClusterName,
263268
VolumeInformer: ctx.InformerFactory.Core().V1().PersistentVolumes(),
@@ -291,6 +296,11 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
291296
csiDriverInformer = ctx.InformerFactory.Storage().V1beta1().CSIDrivers()
292297
}
293298

299+
plugins, err := ProbeAttachableVolumePlugins()
300+
if err != nil {
301+
return nil, true, fmt.Errorf("failed to probe volume plugins when starting attach/detach controller: %v", err)
302+
}
303+
294304
attachDetachController, attachDetachControllerErr :=
295305
attachdetach.NewAttachDetachController(
296306
ctx.ClientBuilder.ClientOrDie("attachdetach-controller"),
@@ -301,7 +311,7 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
301311
csiNodeInformer,
302312
csiDriverInformer,
303313
ctx.Cloud,
304-
ProbeAttachableVolumePlugins(),
314+
plugins,
305315
GetDynamicPluginProber(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
306316
ctx.ComponentConfig.AttachDetachController.DisableAttachDetachReconcilerSync,
307317
ctx.ComponentConfig.AttachDetachController.ReconcilerSyncLoopPeriod.Duration,
@@ -316,17 +326,23 @@ func startAttachDetachController(ctx ControllerContext) (http.Handler, bool, err
316326

317327
func startVolumeExpandController(ctx ControllerContext) (http.Handler, bool, error) {
318328
if utilfeature.DefaultFeatureGate.Enabled(features.ExpandPersistentVolumes) {
329+
plugins, err := ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration)
330+
if err != nil {
331+
return nil, true, fmt.Errorf("failed to probe volume plugins when starting volume expand controller: %v", err)
332+
}
333+
csiTranslator := csitrans.New()
319334
expandController, expandControllerErr := expand.NewExpandController(
320335
ctx.ClientBuilder.ClientOrDie("expand-controller"),
321336
ctx.InformerFactory.Core().V1().PersistentVolumeClaims(),
322337
ctx.InformerFactory.Core().V1().PersistentVolumes(),
323338
ctx.InformerFactory.Storage().V1().StorageClasses(),
324339
ctx.Cloud,
325-
ProbeExpandableVolumePlugins(ctx.ComponentConfig.PersistentVolumeBinderController.VolumeConfiguration),
326-
csitrans.New())
340+
plugins,
341+
csiTranslator,
342+
csimigration.NewPluginManager(csiTranslator))
327343

328344
if expandControllerErr != nil {
329-
return nil, true, fmt.Errorf("failed to start volume expand controller : %v", expandControllerErr)
345+
return nil, true, fmt.Errorf("failed to start volume expand controller: %v", expandControllerErr)
330346
}
331347
go expandController.Run(ctx.Stop)
332348
return nil, true, nil

‎cmd/kube-controller-manager/app/plugins.go‎

Lines changed: 21 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -57,18 +57,21 @@ import (
5757
// detach controller.
5858
// The list of plugins is manually compiled. This code and the plugin
5959
// initialization code for kubelet really, really need a through refactor.
60-
func ProbeAttachableVolumePlugins() []volume.VolumePlugin {
60+
func ProbeAttachableVolumePlugins() ([]volume.VolumePlugin, error) {
61+
var err error
6162
allPlugins := []volume.VolumePlugin{}
62-
63-
allPlugins = appendAttachableLegacyProviderVolumes(allPlugins)
63+
allPlugins, err = appendAttachableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
64+
if err != nil {
65+
return allPlugins, err
66+
}
6467
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
6568
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
6669
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
6770
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
6871
allPlugins = append(allPlugins, iscsi.ProbeVolumePlugins()...)
6972
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
7073
allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
71-
return allPlugins
74+
return allPlugins, nil
7275
}
7376

7477
// GetDynamicPluginProber gets the probers of dynamically discoverable plugins
@@ -79,23 +82,26 @@ func GetDynamicPluginProber(config persistentvolumeconfig.VolumeConfiguration) v
7982
}
8083

8184
// ProbeExpandableVolumePlugins returns volume plugins which are expandable
82-
func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) []volume.VolumePlugin {
85+
func ProbeExpandableVolumePlugins(config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) {
86+
var err error
8387
allPlugins := []volume.VolumePlugin{}
84-
85-
allPlugins = appendExpandableLegacyProviderVolumes(allPlugins)
88+
allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
89+
if err != nil {
90+
return allPlugins, err
91+
}
8692
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
8793
allPlugins = append(allPlugins, glusterfs.ProbeVolumePlugins()...)
8894
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
8995
allPlugins = append(allPlugins, scaleio.ProbeVolumePlugins()...)
9096
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
9197
allPlugins = append(allPlugins, fc.ProbeVolumePlugins()...)
92-
return allPlugins
98+
return allPlugins, nil
9399
}
94100

95101
// ProbeControllerVolumePlugins collects all persistent volume plugins into an
96102
// easy to use list. Only volume plugins that implement any of
97103
// provisioner/recycler/deleter interface should be returned.
98-
func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) []volume.VolumePlugin {
104+
func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persistentvolumeconfig.VolumeConfiguration) ([]volume.VolumePlugin, error) {
99105
allPlugins := []volume.VolumePlugin{}
100106

101107
// The list of plugins to probe is decided by this binary, not
@@ -131,7 +137,11 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste
131137
// add rbd provisioner
132138
allPlugins = append(allPlugins, rbd.ProbeVolumePlugins()...)
133139
allPlugins = append(allPlugins, quobyte.ProbeVolumePlugins()...)
134-
allPlugins = appendLegacyProviderVolumes(allPlugins)
140+
var err error
141+
allPlugins, err = appendExpandableLegacyProviderVolumes(allPlugins, utilfeature.DefaultFeatureGate)
142+
if err != nil {
143+
return allPlugins, err
144+
}
135145

136146
allPlugins = append(allPlugins, flocker.ProbeVolumePlugins()...)
137147
allPlugins = append(allPlugins, portworx.ProbeVolumePlugins()...)
@@ -143,7 +153,7 @@ func ProbeControllerVolumePlugins(cloud cloudprovider.Interface, config persiste
143153
allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
144154
}
145155

146-
return allPlugins
156+
return allPlugins, nil
147157
}
148158

149159
// AttemptToLoadRecycler tries decoding a pod from a filepath for use as a recycler for a volume.

‎cmd/kube-controller-manager/app/plugins_providers.go‎

Lines changed: 65 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -19,34 +19,84 @@ limitations under the License.
1919
package app
2020

2121
import (
22+
"k8s.io/component-base/featuregate"
23+
"k8s.io/csi-translation-lib/plugins"
24+
"k8s.io/klog"
25+
"k8s.io/kubernetes/pkg/features"
2226
"k8s.io/kubernetes/pkg/volume"
2327
"k8s.io/kubernetes/pkg/volume/awsebs"
2428
"k8s.io/kubernetes/pkg/volume/azure_dd"
2529
"k8s.io/kubernetes/pkg/volume/azure_file"
2630
"k8s.io/kubernetes/pkg/volume/cinder"
31+
"k8s.io/kubernetes/pkg/volume/csimigration"
2732
"k8s.io/kubernetes/pkg/volume/gcepd"
2833
"k8s.io/kubernetes/pkg/volume/vsphere_volume"
2934
)
3035

31-
func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin {
32-
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
33-
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
34-
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
35-
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
36+
type probeFn func() []volume.VolumePlugin
37+
38+
func appendPluginBasedOnMigrationFeatureFlags(plugins []volume.VolumePlugin, inTreePluginName string, featureGate featuregate.FeatureGate, pluginMigration, pluginMigrationComplete featuregate.Feature, fn probeFn) ([]volume.VolumePlugin, error) {
39+
// Skip appending the in-tree plugin to the list of plugins to be probed/initialized
40+
// if the CSIMigration feature flag and plugin specific feature flag indicating
41+
// CSI migration is complete
42+
err := csimigration.CheckMigrationFeatureFlags(featureGate, pluginMigration, pluginMigrationComplete)
43+
if err != nil {
44+
klog.Warningf("Unexpected CSI Migration Feature Flags combination detected: %v. CSI Migration may not take effect", err)
45+
// TODO: fail and return here once alpha only tests can set the feature flags for a plugin correctly
46+
}
47+
if featureGate.Enabled(features.CSIMigration) && featureGate.Enabled(pluginMigration) && featureGate.Enabled(pluginMigrationComplete) {
48+
klog.Infof("Skip registration of plugin %s since feature flag %v is enabled", inTreePluginName, pluginMigrationComplete)
49+
return plugins, nil
50+
}
51+
plugins = append(plugins, fn()...)
52+
return plugins, nil
53+
}
54+
55+
type pluginInfo struct {
56+
pluginMigrationFeature featuregate.Feature
57+
pluginMigrationCompleteFeature featuregate.Feature
58+
pluginProbeFunction probeFn
59+
}
60+
61+
func appendAttachableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
62+
pluginMigrationStatus := make(map[string]pluginInfo)
63+
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins}
64+
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins}
65+
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins}
66+
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins}
67+
68+
var err error
69+
for pluginName, pluginInfo := range pluginMigrationStatus {
70+
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction)
71+
if err != nil {
72+
return allPlugins, err
73+
}
74+
}
75+
3676
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
37-
return allPlugins
77+
return allPlugins, nil
3878
}
3979

40-
func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin {
41-
return appendLegacyProviderVolumes(allPlugins)
80+
func appendExpandableLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
81+
return appendLegacyProviderVolumes(allPlugins, featureGate)
4282
}
4383

44-
func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin) []volume.VolumePlugin {
45-
allPlugins = append(allPlugins, awsebs.ProbeVolumePlugins()...)
46-
allPlugins = append(allPlugins, azure_dd.ProbeVolumePlugins()...)
47-
allPlugins = append(allPlugins, azure_file.ProbeVolumePlugins()...)
48-
allPlugins = append(allPlugins, cinder.ProbeVolumePlugins()...)
49-
allPlugins = append(allPlugins, gcepd.ProbeVolumePlugins()...)
84+
func appendLegacyProviderVolumes(allPlugins []volume.VolumePlugin, featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
85+
pluginMigrationStatus := make(map[string]pluginInfo)
86+
pluginMigrationStatus[plugins.AWSEBSInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAWS, pluginMigrationCompleteFeature: features.CSIMigrationAWSComplete, pluginProbeFunction: awsebs.ProbeVolumePlugins}
87+
pluginMigrationStatus[plugins.GCEPDInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationGCE, pluginMigrationCompleteFeature: features.CSIMigrationGCEComplete, pluginProbeFunction: gcepd.ProbeVolumePlugins}
88+
pluginMigrationStatus[plugins.CinderInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationOpenStack, pluginMigrationCompleteFeature: features.CSIMigrationOpenStackComplete, pluginProbeFunction: cinder.ProbeVolumePlugins}
89+
pluginMigrationStatus[plugins.AzureDiskInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureDisk, pluginMigrationCompleteFeature: features.CSIMigrationAzureDiskComplete, pluginProbeFunction: azure_dd.ProbeVolumePlugins}
90+
pluginMigrationStatus[plugins.AzureFileInTreePluginName] = pluginInfo{pluginMigrationFeature: features.CSIMigrationAzureFile, pluginMigrationCompleteFeature: features.CSIMigrationAzureFileComplete, pluginProbeFunction: azure_file.ProbeVolumePlugins}
91+
92+
var err error
93+
for pluginName, pluginInfo := range pluginMigrationStatus {
94+
allPlugins, err = appendPluginBasedOnMigrationFeatureFlags(allPlugins, pluginName, featureGate, pluginInfo.pluginMigrationFeature, pluginInfo.pluginMigrationCompleteFeature, pluginInfo.pluginProbeFunction)
95+
if err != nil {
96+
return allPlugins, err
97+
}
98+
}
99+
50100
allPlugins = append(allPlugins, vsphere_volume.ProbeVolumePlugins()...)
51-
return allPlugins
101+
return allPlugins, nil
52102
}

‎cmd/kubelet/app/BUILD‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -86,6 +86,7 @@ go_library(
8686
"//pkg/volume/cinder:go_default_library",
8787
"//pkg/volume/configmap:go_default_library",
8888
"//pkg/volume/csi:go_default_library",
89+
"//pkg/volume/csimigration:go_default_library",
8990
"//pkg/volume/downwardapi:go_default_library",
9091
"//pkg/volume/emptydir:go_default_library",
9192
"//pkg/volume/fc:go_default_library",
@@ -138,8 +139,10 @@ go_library(
138139
"//staging/src/k8s.io/client-go/util/keyutil:go_default_library",
139140
"//staging/src/k8s.io/cloud-provider:go_default_library",
140141
"//staging/src/k8s.io/component-base/cli/flag:go_default_library",
142+
"//staging/src/k8s.io/component-base/featuregate:go_default_library",
141143
"//staging/src/k8s.io/component-base/version:go_default_library",
142144
"//staging/src/k8s.io/component-base/version/verflag:go_default_library",
145+
"//staging/src/k8s.io/csi-translation-lib/plugins:go_default_library",
143146
"//staging/src/k8s.io/kubelet/config/v1beta1:go_default_library",
144147
"//vendor/github.com/coreos/go-systemd/daemon:go_default_library",
145148
"//vendor/github.com/spf13/cobra:go_default_library",

‎cmd/kubelet/app/plugins.go‎

Lines changed: 9 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@ import (
2222
_ "k8s.io/kubernetes/pkg/credentialprovider/aws"
2323
_ "k8s.io/kubernetes/pkg/credentialprovider/azure"
2424
_ "k8s.io/kubernetes/pkg/credentialprovider/gcp"
25+
26+
"k8s.io/component-base/featuregate"
2527
"k8s.io/utils/exec"
2628

2729
// Volume plugins
@@ -53,7 +55,7 @@ import (
5355
)
5456

5557
// ProbeVolumePlugins collects all volume plugins into an easy to use list.
56-
func ProbeVolumePlugins() []volume.VolumePlugin {
58+
func ProbeVolumePlugins(featureGate featuregate.FeatureGate) ([]volume.VolumePlugin, error) {
5759
allPlugins := []volume.VolumePlugin{}
5860

5961
// The list of plugins to probe is decided by the kubelet binary, not
@@ -62,7 +64,11 @@ func ProbeVolumePlugins() []volume.VolumePlugin {
6264
//
6365
// Kubelet does not currently need to configure volume plugins.
6466
// If/when it does, see kube-controller-manager/app/plugins.go for example of using volume.VolumeConfig
65-
allPlugins = appendLegacyProviderVolumes(allPlugins)
67+
var err error
68+
allPlugins, err = appendLegacyProviderVolumes(allPlugins, featureGate)
69+
if err != nil {
70+
return allPlugins, err
71+
}
6672
allPlugins = append(allPlugins, emptydir.ProbeVolumePlugins()...)
6773
allPlugins = append(allPlugins, git_repo.ProbeVolumePlugins()...)
6874
allPlugins = append(allPlugins, hostpath.ProbeVolumePlugins(volume.VolumeConfig{})...)
@@ -83,7 +89,7 @@ func ProbeVolumePlugins() []volume.VolumePlugin {
8389
allPlugins = append(allPlugins, local.ProbeVolumePlugins()...)
8490
allPlugins = append(allPlugins, storageos.ProbeVolumePlugins()...)
8591
allPlugins = append(allPlugins, csi.ProbeVolumePlugins()...)
86-
return allPlugins
92+
return allPlugins, nil
8793
}
8894

8995
// GetDynamicPluginProber gets the probers of dynamically discoverable plugins

0 commit comments

Comments
 (0)