Merge pull request kubernetes#128197 from aojea/extract_provider_flags

disable cloud-provider code from kube-controller-manager

Author: k8s-ci-robot

cluster/gce/gci/configure-helper.sh

@@ -2224,9 +2224,6 @@ function start-kube-controller-manager {
   if [[ -n "${SERVICE_CLUSTER_IP_RANGE:-}" ]]; then
     params+=("--service-cluster-ip-range=${SERVICE_CLUSTER_IP_RANGE}")
   fi
-  if [[ -n "${CONCURRENT_SERVICE_SYNCS:-}" ]]; then
-    params+=("--concurrent-service-syncs=${CONCURRENT_SERVICE_SYNCS}")
-  fi
   if [[ "${NETWORK_PROVIDER:-}" == "kubenet" ]]; then
     params+=("--allocate-node-cidrs=true")
   elif [[ -n "${ALLOCATE_NODE_CIDRS:-}" ]]; then
@@ -2235,10 +2232,6 @@ function start-kube-controller-manager {
   if [[ -n "${TERMINATED_POD_GC_THRESHOLD:-}" ]]; then
     params+=("--terminated-pod-gc-threshold=${TERMINATED_POD_GC_THRESHOLD}")
   fi
-  if [[ "${ENABLE_IP_ALIASES:-}" == 'true' ]]; then
-    params+=("--cidr-allocator-type=${NODE_IPAM_MODE}")
-    params+=("--configure-cloud-routes=false")
-  fi
   if [[ -n "${FEATURE_GATES:-}" ]]; then
     params+=("--feature-gates=${FEATURE_GATES}")
   fi

cmd/kube-controller-manager/app/cloudproviders.go

@@ -53,7 +53,6 @@ import (
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
-	cloudprovider "k8s.io/cloud-provider"
 	cliflag "k8s.io/component-base/cli/flag"
 	"k8s.io/component-base/cli/globalflag"
 	"k8s.io/component-base/configz"
@@ -95,16 +94,6 @@ const (
 	ConfigzName = "kubecontrollermanager.config.k8s.io"
 )
 
-// ControllerLoopMode is the kube-controller-manager's mode of running controller loops that are cloud provider dependent
-type ControllerLoopMode int
-
-const (
-	// IncludeCloudLoops means the kube-controller-manager include the controller loops that are cloud provider dependent
-	IncludeCloudLoops ControllerLoopMode = iota
-	// ExternalLoops means the kube-controller-manager exclude the controller loops that are cloud provider dependent
-	ExternalLoops
-)
-
 // NewControllerManagerCommand creates a *cobra.Command object with default parameters
 func NewControllerManagerCommand() *cobra.Command {
 	_, _ = utilversion.DefaultComponentGlobalsRegistry.ComponentGlobalsOrRegister(
@@ -396,15 +385,6 @@ type ControllerContext struct {
 	// requested.
 	RESTMapper *restmapper.DeferredDiscoveryRESTMapper
 
-	// Cloud is the cloud provider interface for the controllers to use.
-	// It must be initialized and ready to use.
-	Cloud cloudprovider.Interface
-
-	// Control for which control loops to be run
-	// IncludeCloudLoops is for a kube-controller-manager running all loops
-	// ExternalLoops is for a kube-controller-manager running with a cloud-controller-manager
-	LoopMode ControllerLoopMode
-
 	// InformersStarted is closed after all of the controllers have been initialized and are running.  After this point it is safe,
 	// for an individual controller to start the shared informers. Before it is closed, they should not.
 	InformersStarted chan struct{}
@@ -644,20 +624,12 @@ func CreateControllerContext(ctx context.Context, s *config.CompletedConfig, roo
 		restMapper.Reset()
 	}, 30*time.Second, ctx.Done())
 
-	cloud, loopMode, err := createCloudProvider(klog.FromContext(ctx), s.ComponentConfig.KubeCloudShared.CloudProvider.Name, s.ComponentConfig.KubeCloudShared.ExternalCloudVolumePlugin,
-		s.ComponentConfig.KubeCloudShared.CloudProvider.CloudConfigFile, s.ComponentConfig.KubeCloudShared.AllowUntaggedCloud, sharedInformers)
-	if err != nil {
-		return ControllerContext{}, err
-	}
-
 	controllerContext := ControllerContext{
 		ClientBuilder:                   clientBuilder,
 		InformerFactory:                 sharedInformers,
 		ObjectOrMetadataInformerFactory: informerfactory.NewInformerFactory(sharedInformers, metadataInformers),
 		ComponentConfig:                 s.ComponentConfig,
 		RESTMapper:                      restMapper,
-		Cloud:                           cloud,
-		LoopMode:                        loopMode,
 		InformersStarted:                make(chan struct{}),
 		ResyncPeriod:                    ResyncPeriod(s),
 		ControllerManagerMetrics:        controllersmetrics.NewControllerManagerMetrics("kube-controller-manager"),
@@ -702,12 +674,6 @@ func StartControllers(ctx context.Context, controllerCtx ControllerContext, cont
 		}
 	}
 
-	// Initialize the cloud provider with a reference to the clientBuilder only after token controller
-	// has started in case the cloud provider uses the client builder.
-	if controllerCtx.Cloud != nil {
-		controllerCtx.Cloud.Initialize(controllerCtx.ClientBuilder, ctx.Done())
-	}
-
 	// Each controller is passed a context where the logger has the name of
 	// the controller set through WithName. That name then becomes the prefix of
 	// of all log messages emitted by that controller.
@@ -751,8 +717,8 @@ func StartController(ctx context.Context, controllerCtx ControllerContext, contr
 		}
 	}
 
-	if controllerDescriptor.IsCloudProviderController() && controllerCtx.LoopMode != IncludeCloudLoops {
-		logger.Info("Skipping a cloud provider controller", "controller", controllerName, "loopMode", controllerCtx.LoopMode)
+	if controllerDescriptor.IsCloudProviderController() {
+		logger.Info("Skipping a cloud provider controller", "controller", controllerName)
 		return nil, nil
 	}
 

cmd/kube-controller-manager/app/controllermanager.go

@@ -225,10 +225,7 @@ func TestNoCloudProviderControllerStarted(t *testing.T) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 
-	controllerCtx := ControllerContext{
-		Cloud:    nil,
-		LoopMode: IncludeCloudLoops,
-	}
+	controllerCtx := ControllerContext{}
 	controllerCtx.ComponentConfig.Generic.Controllers = []string{"*"}
 	for _, controller := range NewControllerDescriptors() {
 		if !controller.IsCloudProviderController() {

cmd/kube-controller-manager/app/controllermanager_test.go

@@ -36,9 +36,6 @@ import (
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/metadata"
 	restclient "k8s.io/client-go/rest"
-	cloudnodelifecyclecontroller "k8s.io/cloud-provider/controllers/nodelifecycle"
-	routecontroller "k8s.io/cloud-provider/controllers/route"
-	servicecontroller "k8s.io/cloud-provider/controllers/service"
 	cpnames "k8s.io/cloud-provider/names"
 	"k8s.io/component-base/featuregate"
 	"k8s.io/controller-manager/controller"
@@ -92,26 +89,8 @@ func newServiceLBControllerDescriptor() *ControllerDescriptor {
 
 func startServiceLBController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
 	logger := klog.FromContext(ctx)
-	if controllerContext.Cloud == nil {
-		logger.Info("Warning: service-controller is set, but no cloud provider specified. Will not configure service controller.")
-		return nil, false, nil
-	}
-
-	serviceController, err := servicecontroller.New(
-		controllerContext.Cloud,
-		controllerContext.ClientBuilder.ClientOrDie("service-controller"),
-		controllerContext.InformerFactory.Core().V1().Services(),
-		controllerContext.InformerFactory.Core().V1().Nodes(),
-		controllerContext.ComponentConfig.KubeCloudShared.ClusterName,
-		utilfeature.DefaultFeatureGate,
-	)
-	if err != nil {
-		// This error shouldn't fail. It lives like this as a legacy.
-		logger.Error(err, "Failed to start service controller.")
-		return nil, false, nil
-	}
-	go serviceController.Run(ctx, int(controllerContext.ComponentConfig.ServiceController.ConcurrentServiceSyncs), controllerContext.ControllerManagerMetrics)
-	return nil, true, nil
+	logger.Info("Warning: service-controller is set, but no cloud provider functionality is available in kube-controller-manger (KEP-2395). Will not configure service controller.")
+	return nil, false, nil
 }
 func newNodeIpamControllerDescriptor() *ControllerDescriptor {
 	return &ControllerDescriptor{
@@ -133,11 +112,7 @@ func startNodeIpamController(ctx context.Context, controllerContext ControllerCo
 
 	if controllerContext.ComponentConfig.KubeCloudShared.CIDRAllocatorType == string(ipam.CloudAllocatorType) {
 		// Cannot run cloud ipam controller if cloud provider is nil (--cloud-provider not set or set to 'external')
-		if controllerContext.Cloud == nil {
-			return nil, false, errors.New("--cidr-allocator-type is set to 'CloudAllocator' but cloud provider is not configured")
-		}
-		// As part of the removal of all the cloud providers from kubernetes, this support will be removed as well
-		klog.Warningf("DEPRECATED: 'CloudAllocator' bas been deprecated and will be removed in a future release.")
+		return nil, false, errors.New("--cidr-allocator-type is set to 'CloudAllocator' but cloud provider is not configured")
 	}
 
 	clusterCIDRs, err := validateCIDRs(controllerContext.ComponentConfig.KubeCloudShared.ClusterCIDR)
@@ -182,7 +157,7 @@ func startNodeIpamController(ctx context.Context, controllerContext ControllerCo
 	nodeIpamController, err := nodeipamcontroller.NewNodeIpamController(
 		ctx,
 		controllerContext.InformerFactory.Core().V1().Nodes(),
-		controllerContext.Cloud,
+		nil, // no cloud provider on kube-controller-manager since v1.31 (KEP-2395)
 		controllerContext.ClientBuilder.ClientOrDie("node-controller"),
 		clusterCIDRs,
 		serviceCIDR,
@@ -266,27 +241,8 @@ func newCloudNodeLifecycleControllerDescriptor() *ControllerDescriptor {
 
 func startCloudNodeLifecycleController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
 	logger := klog.FromContext(ctx)
-	if controllerContext.Cloud == nil {
-		logger.Info("Warning: node-controller is set, but no cloud provider specified. Will not configure node lifecyle controller.")
-		return nil, false, nil
-	}
-
-	cloudNodeLifecycleController, err := cloudnodelifecyclecontroller.NewCloudNodeLifecycleController(
-		controllerContext.InformerFactory.Core().V1().Nodes(),
-		// cloud node lifecycle controller uses existing cluster role from node-controller
-		controllerContext.ClientBuilder.ClientOrDie("node-controller"),
-		controllerContext.Cloud,
-		controllerContext.ComponentConfig.KubeCloudShared.NodeMonitorPeriod.Duration,
-	)
-	if err != nil {
-		// the controller manager should continue to run if the "Instances" interface is not
-		// supported, though it's unlikely for a cloud provider to not support it
-		logger.Error(err, "Failed to start cloud node lifecycle controller")
-		return nil, false, nil
-	}
-
-	go cloudNodeLifecycleController.Run(ctx, controllerContext.ControllerManagerMetrics)
-	return nil, true, nil
+	logger.Info("Warning: node-controller is set, but no cloud provider functionality is available in kube-controller-manger (KEP-2395). Will not configure node lifecyle controller.")
+	return nil, false, nil
 }
 
 func newNodeRouteControllerDescriptor() *ControllerDescriptor {
@@ -300,32 +256,8 @@ func newNodeRouteControllerDescriptor() *ControllerDescriptor {
 
 func startNodeRouteController(ctx context.Context, controllerContext ControllerContext, controllerName string) (controller.Interface, bool, error) {
 	logger := klog.FromContext(ctx)
-	if !controllerContext.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs || !controllerContext.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes {
-		logger.Info("Will not configure cloud provider routes for allocate-node-cidrs", "CIDRs", controllerContext.ComponentConfig.KubeCloudShared.AllocateNodeCIDRs, "routes", controllerContext.ComponentConfig.KubeCloudShared.ConfigureCloudRoutes)
-		return nil, false, nil
-	}
-	if controllerContext.Cloud == nil {
-		logger.Info("Warning: configure-cloud-routes is set, but no cloud provider specified. Will not configure cloud provider routes.")
-		return nil, false, nil
-	}
-	routes, ok := controllerContext.Cloud.Routes()
-	if !ok {
-		logger.Info("Warning: configure-cloud-routes is set, but cloud provider does not support routes. Will not configure cloud provider routes.")
-		return nil, false, nil
-	}
-
-	clusterCIDRs, err := validateCIDRs(controllerContext.ComponentConfig.KubeCloudShared.ClusterCIDR)
-	if err != nil {
-		return nil, false, err
-	}
-
-	routeController := routecontroller.New(routes,
-		controllerContext.ClientBuilder.ClientOrDie("route-controller"),
-		controllerContext.InformerFactory.Core().V1().Nodes(),
-		controllerContext.ComponentConfig.KubeCloudShared.ClusterName,
-		clusterCIDRs)
-	go routeController.Run(ctx, controllerContext.ComponentConfig.KubeCloudShared.RouteReconciliationPeriod.Duration, controllerContext.ControllerManagerMetrics)
-	return nil, true, nil
+	logger.Info("Warning: configure-cloud-routes is set, but no cloud provider functionality is available in kube-controller-manger (KEP-2395). Will not configure cloud provider routes.")
+	return nil, false, nil
 }
 
 func newPersistentVolumeBinderControllerDescriptor() *ControllerDescriptor {

cmd/kube-controller-manager/app/core.go

@@ -33,6 +33,7 @@ import (
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/record"
+	cloudprovider "k8s.io/cloud-provider"
 	cpnames "k8s.io/cloud-provider/names"
 	cpoptions "k8s.io/cloud-provider/options"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -50,6 +51,7 @@ import (
 	kubectrlmgrconfigscheme "k8s.io/kubernetes/pkg/controller/apis/config/scheme"
 	"k8s.io/kubernetes/pkg/controller/garbagecollector"
 	garbagecollectorconfig "k8s.io/kubernetes/pkg/controller/garbagecollector/config"
+	"k8s.io/kubernetes/pkg/controller/nodeipam/ipam"
 	netutils "k8s.io/utils/net"
 
 	// add the kubernetes feature gates
@@ -452,6 +454,16 @@ func (s *KubeControllerManagerOptions) Validate(allControllers []string, disable
 	errs = append(errs, s.Metrics.Validate()...)
 	errs = append(errs, utilversion.ValidateKubeEffectiveVersion(s.ComponentGlobalsRegistry.EffectiveVersionFor(utilversion.DefaultKubeComponent)))
 
+	// in-tree cloud providers are disabled since v1.31 (KEP-2395)
+	if len(s.KubeCloudShared.CloudProvider.Name) > 0 && !cloudprovider.IsExternal(s.KubeCloudShared.CloudProvider.Name) {
+		cloudprovider.DisableWarningForProvider(s.KubeCloudShared.CloudProvider.Name)
+		errs = append(errs, cloudprovider.ErrorForDisabledProvider(s.KubeCloudShared.CloudProvider.Name))
+	}
+
+	if len(s.KubeCloudShared.CIDRAllocatorType) > 0 && s.KubeCloudShared.CIDRAllocatorType != string(ipam.RangeAllocatorType) {
+		errs = append(errs, fmt.Errorf("built-in cloud providers are disabled. The ipam %s is not available", s.KubeCloudShared.CIDRAllocatorType))
+	}
+
 	// TODO: validate component config, master and kubeconfig
 
 	return utilerrors.NewAggregate(errs)