Expose flag to explicitly disable cloud-provider=external kubelet argument (#60)

anmazzotti · web-flow · commit 4a936a9b13c1 · 2023-10-12T09:23:16.000-04:00
* Expose flag to explicitly set cloud-provider=external kubelet argument
diff --git a/bootstrap/api/v1beta1/kthreesconfig_types.go b/bootstrap/api/v1beta1/kthreesconfig_types.go
@@ -103,6 +103,10 @@ type KThreesServerConfig struct {
 	// DisableComponents  specifies extra commands to run before k3s setup runs
 	// +optional
 	DisableComponents []string `json:"disableComponents,omitempty"`
+
+	// DisableExternalCloudProvider suppresses the 'cloud-provider=external' kubelet argument. (default: false)
+	// +optional
+	DisableExternalCloudProvider bool `json:"disableExternalCloudProvider,omitempty"`
 }
 
 type KThreesAgentConfig struct {
diff --git a/bootstrap/api/v1beta1/zz_generated.deepcopy.go b/bootstrap/api/v1beta1/zz_generated.deepcopy.go
diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml
@@ -166,6 +166,10 @@ spec:
                     items:
                       type: string
                     type: array
+                  disableExternalCloudProvider:
+                    description: 'DisableExternalCloudProvider suppresses the ''cloud-provider=external''
+                      kubelet argument. (default: false)'
+                    type: boolean
                   httpsListenPort:
                     description: 'HTTPSListenPort HTTPS listen port (default: 6443)'
                     type: string
diff --git a/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/bootstrap/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml
@@ -180,6 +180,11 @@ spec:
                             items:
                               type: string
                             type: array
+                          disableExternalCloudProvider:
+                            description: 'DisableExternalCloudProvider suppresses
+                              the ''cloud-provider=external'' kubelet argument. (default:
+                              false)'
+                            type: boolean
                           httpsListenPort:
                             description: 'HTTPSListenPort HTTPS listen port (default:
                               6443)'
diff --git a/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/bootstrap/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml
@@ -246,6 +246,11 @@ spec:
                         items:
                           type: string
                         type: array
+                      disableExternalCloudProvider:
+                        description: 'DisableExternalCloudProvider suppresses the
+                          ''cloud-provider=external'' kubelet argument. (default:
+                          false)'
+                        type: boolean
                       httpsListenPort:
                         description: 'HTTPSListenPort HTTPS listen port (default:
                           6443)'
diff --git a/bootstrap/config/manager/kustomization.yaml b/bootstrap/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/cluster-api-provider-k3s/cluster-api-k3s/bootstrap-controller
-  newTag: v0.2.0
+  newTag: v0.1.7
diff --git a/bootstrap/controllers/kthreesconfig_controller.go b/bootstrap/controllers/kthreesconfig_controller.go
@@ -290,7 +290,7 @@ func (r *KThreesConfigReconciler) joinWorker(ctx context.Context, scope *Scope)
 		return err
 	}
 
-	configStruct := k3s.GenerateWorkerConfig(serverURL, tokn, scope.Config.Spec.AgentConfig)
+	configStruct := k3s.GenerateWorkerConfig(serverURL, tokn, scope.Config.Spec.ServerConfig, scope.Config.Spec.AgentConfig)
 
 	b, err := kubeyaml.Marshal(configStruct)
 	if err != nil {
diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigs.yaml
@@ -166,6 +166,10 @@ spec:
                     items:
                       type: string
                     type: array
+                  disableExternalCloudProvider:
+                    description: 'DisableExternalCloudProvider suppresses the ''cloud-provider=external''
+                      kubelet argument. (default: false)'
+                    type: boolean
                   httpsListenPort:
                     description: 'HTTPSListenPort HTTPS listen port (default: 6443)'
                     type: string
@@ -181,6 +185,12 @@ spec:
                     items:
                       type: string
                     type: array
+                  kubeSchedulerArgs:
+                    description: KubeSchedulerArgs is a customized flag for kube-scheduler
+                      process
+                    items:
+                      type: string
+                    type: array
                   serviceCidr:
                     description: 'ServiceCidr Network CIDR to use for services IPs
                       (default: "10.43.0.0/16")'
diff --git a/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml b/controlplane/config/crd/bases/bootstrap.cluster.x-k8s.io_kthreesconfigtemplates.yaml
@@ -180,6 +180,11 @@ spec:
                             items:
                               type: string
                             type: array
+                          disableExternalCloudProvider:
+                            description: 'DisableExternalCloudProvider suppresses
+                              the ''cloud-provider=external'' kubelet argument. (default:
+                              false)'
+                            type: boolean
                           httpsListenPort:
                             description: 'HTTPSListenPort HTTPS listen port (default:
                               6443)'
@@ -196,6 +201,12 @@ spec:
                             items:
                               type: string
                             type: array
+                          kubeSchedulerArgs:
+                            description: KubeSchedulerArgs is a customized flag for
+                              kube-scheduler process
+                            items:
+                              type: string
+                            type: array
                           serviceCidr:
                             description: 'ServiceCidr Network CIDR to use for services
                               IPs (default: "10.43.0.0/16")'
diff --git a/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml b/controlplane/config/crd/bases/controlplane.cluster.x-k8s.io_kthreescontrolplanes.yaml
@@ -246,6 +246,11 @@ spec:
                         items:
                           type: string
                         type: array
+                      disableExternalCloudProvider:
+                        description: 'DisableExternalCloudProvider suppresses the
+                          ''cloud-provider=external'' kubelet argument. (default:
+                          false)'
+                        type: boolean
                       httpsListenPort:
                         description: 'HTTPSListenPort HTTPS listen port (default:
                           6443)'
@@ -262,6 +267,12 @@ spec:
                         items:
                           type: string
                         type: array
+                      kubeSchedulerArgs:
+                        description: KubeSchedulerArgs is a customized flag for kube-scheduler
+                          process
+                        items:
+                          type: string
+                        type: array
                       serviceCidr:
                         description: 'ServiceCidr Network CIDR to use for services
                           IPs (default: "10.43.0.0/16")'
diff --git a/controlplane/config/manager/kustomization.yaml b/controlplane/config/manager/kustomization.yaml
@@ -5,4 +5,4 @@ kind: Kustomization
 images:
 - name: controller
   newName: ghcr.io/cluster-api-provider-k3s/cluster-api-k3s/controlplane-controller
-  newTag: v0.2.0
+  newTag: v0.1.7
diff --git a/pkg/k3s/config.go b/pkg/k3s/config.go
@@ -40,12 +40,13 @@ type K3sAgentConfig struct {
 }
 
 func GenerateInitControlPlaneConfig(controlPlaneEndpoint string, token string, serverConfig bootstrapv1.KThreesServerConfig, agentConfig bootstrapv1.KThreesAgentConfig) K3sServerConfig {
+	kubeletExtraArgs := getKubeletExtraArgs(serverConfig)
 	k3sServerConfig := K3sServerConfig{
-		DisableCloudController:    true,
+		DisableCloudController:    !serverConfig.DisableExternalCloudProvider,
 		ClusterInit:               true,
 		KubeAPIServerArgs:         append(serverConfig.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()),
 		TLSSan:                    append(serverConfig.TLSSan, controlPlaneEndpoint),
-		KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, "cloud-provider=external"),
+		KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, kubeletExtraArgs...),
 		KubeSchedulerArgs:         serverConfig.KubeSchedulerArgs,
 		BindAddress:               serverConfig.BindAddress,
 		HTTPSListenPort:           serverConfig.HTTPSListenPort,
@@ -60,7 +61,7 @@ func GenerateInitControlPlaneConfig(controlPlaneEndpoint string, token string, s
 
 	k3sServerConfig.K3sAgentConfig = K3sAgentConfig{
 		Token:           token,
-		KubeletArgs:     append(agentConfig.KubeletArgs, "cloud-provider=external"),
+		KubeletArgs:     append(agentConfig.KubeletArgs, kubeletExtraArgs...),
 		NodeLabels:      agentConfig.NodeLabels,
 		NodeTaints:      agentConfig.NodeTaints,
 		PrivateRegistry: agentConfig.PrivateRegistry,
@@ -72,11 +73,12 @@ func GenerateInitControlPlaneConfig(controlPlaneEndpoint string, token string, s
 }
 
 func GenerateJoinControlPlaneConfig(serverURL string, token string, controlplaneendpoint string, serverConfig bootstrapv1.KThreesServerConfig, agentConfig bootstrapv1.KThreesAgentConfig) K3sServerConfig {
+	kubeletExtraArgs := getKubeletExtraArgs(serverConfig)
 	k3sServerConfig := K3sServerConfig{
-		DisableCloudController:    true,
+		DisableCloudController:    !serverConfig.DisableExternalCloudProvider,
 		KubeAPIServerArgs:         append(serverConfig.KubeAPIServerArgs, "anonymous-auth=true", getTLSCipherSuiteArg()),
 		TLSSan:                    append(serverConfig.TLSSan, controlplaneendpoint),
-		KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, "cloud-provider=external"),
+		KubeControllerManagerArgs: append(serverConfig.KubeControllerManagerArgs, kubeletExtraArgs...),
 		KubeSchedulerArgs:         serverConfig.KubeSchedulerArgs,
 		BindAddress:               serverConfig.BindAddress,
 		HTTPSListenPort:           serverConfig.HTTPSListenPort,
@@ -92,7 +94,7 @@ func GenerateJoinControlPlaneConfig(serverURL string, token string, controlplane
 	k3sServerConfig.K3sAgentConfig = K3sAgentConfig{
 		Token:           token,
 		Server:          serverURL,
-		KubeletArgs:     append(agentConfig.KubeletArgs, "cloud-provider=external"),
+		KubeletArgs:     append(agentConfig.KubeletArgs, kubeletExtraArgs...),
 		NodeLabels:      agentConfig.NodeLabels,
 		NodeTaints:      agentConfig.NodeTaints,
 		PrivateRegistry: agentConfig.PrivateRegistry,
@@ -103,11 +105,12 @@ func GenerateJoinControlPlaneConfig(serverURL string, token string, controlplane
 	return k3sServerConfig
 }
 
-func GenerateWorkerConfig(serverURL string, token string, agentConfig bootstrapv1.KThreesAgentConfig) K3sAgentConfig {
+func GenerateWorkerConfig(serverURL string, token string, serverConfig bootstrapv1.KThreesServerConfig, agentConfig bootstrapv1.KThreesAgentConfig) K3sAgentConfig {
+	kubeletExtraArgs := getKubeletExtraArgs(serverConfig)
 	return K3sAgentConfig{
 		Server:          serverURL,
 		Token:           token,
-		KubeletArgs:     append(agentConfig.KubeletArgs, "cloud-provider=external"),
+		KubeletArgs:     append(agentConfig.KubeletArgs, kubeletExtraArgs...),
 		NodeLabels:      agentConfig.NodeLabels,
 		NodeTaints:      agentConfig.NodeTaints,
 		PrivateRegistry: agentConfig.PrivateRegistry,
@@ -152,3 +155,11 @@ func getTLSCipherSuiteArg() string {
 
 	return fmt.Sprintf("tls-cipher-suites=%s", ciphersList)
 }
+
+func getKubeletExtraArgs(serverConfig bootstrapv1.KThreesServerConfig) []string {
+	kubeletExtraArgs := []string{}
+	if !serverConfig.DisableExternalCloudProvider {
+		kubeletExtraArgs = append(kubeletExtraArgs, "cloud-provider=external")
+	}
+	return kubeletExtraArgs
+}

Original file line number	Diff line number	Diff line change
`@@ -290,7 +290,7 @@ func (r KThreesConfigReconciler) joinWorker(ctx context.Context, scope Scope)`
`290`	`290`	`return err`
`291`	`291`	`}`
`292`	`292`
`293`		`- configStruct := k3s.GenerateWorkerConfig(serverURL, tokn, scope.Config.Spec.AgentConfig)`
	`293`	`+ configStruct := k3s.GenerateWorkerConfig(serverURL, tokn, scope.Config.Spec.ServerConfig, scope.Config.Spec.AgentConfig)`
`294`	`294`
`295`	`295`	`b, err := kubeyaml.Marshal(configStruct)`
`296`	`296`	`if err != nil {`