[node-labeler] Refactor node labeling to use taints instead of labels (#20652)

* [node-labeler] Refactor node labeling to use taints instead of labels

* [agent-smith] Add toleration to daemonset

* Add workspace component tolerations to various Gitpod components if it running in Full installation

* Apply suggestions from code review

Co-authored-by: Kyle Brennan <[email protected]>

* Update components/node-labeler/cmd/run.go

Co-authored-by: Kyle Brennan <[email protected]>

---------

Co-authored-by: Kyle Brennan <[email protected]>

Author: iQQBot

components/node-labeler/cmd/run.go

@@ -0,0 +1,33 @@
+// Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+// Licensed under the GNU Affero General Public License (AGPL).
+// See License.AGPL.txt in the project root for license information.
+
+package common
+
+import (
+	configv1 "github.com/gitpod-io/gitpod/installer/pkg/config/v1"
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	RegistryFacadeTaintKey = "gitpod.io/registry-facade-not-ready"
+	WsDaemonTaintKey       = "gitpod.io/ws-daemon-not-ready"
+)
+
+func WithTolerationWorkspaceComponentNotReady(ctx *RenderContext) []corev1.Toleration {
+	if ctx.Config.Kind != configv1.InstallationFull {
+		return []corev1.Toleration{}
+	}
+	return []corev1.Toleration{
+		{
+			Key:      RegistryFacadeTaintKey,
+			Operator: corev1.TolerationOpExists,
+			Effect:   corev1.TaintEffectNoSchedule,
+		},
+		{
+			Key:      WsDaemonTaintKey,
+			Operator: corev1.TolerationOpExists,
+			Effect:   corev1.TaintEffectNoSchedule,
+		},
+	}
+}

install/installer/pkg/common/toleration.go

@@ -105,6 +105,12 @@ func daemonset(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 						common.CAVolume(),
 					},
+					Tolerations: []corev1.Toleration{
+						{
+							Effect:   corev1.TaintEffectNoSchedule,
+							Operator: corev1.TolerationOpExists,
+						},
+					},
 				},
 			},
 			UpdateStrategy: common.DaemonSetRolloutStrategy(),

install/installer/pkg/components/agent-smith/daemonset.go

@@ -168,6 +168,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								FailureThreshold:    3,
 							},
 						}, *common.KubeRBACProxyContainer(ctx)},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

install/installer/pkg/components/blobserve/deployment.go

@@ -88,6 +88,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 			},
 		}, *common.KubeRBACProxyContainer(ctx),
 		},
+		Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 	}
 
 	err = common.AddStorageMounts(ctx, &podSpec, Component)

install/installer/pkg/components/content-service/deployment.go

@@ -86,6 +86,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								TimeoutSeconds:   1,
 							},
 						}},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

install/installer/pkg/components/dashboard/deployment.go

@@ -25,6 +25,15 @@ var Helm = common.CompositeHelmFunc(
 			return nil, err
 		}
 
+		tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
+		if err != nil {
+			return nil, err
+		}
+		tolerationsTemplate, err := helm.KeyFileValue("mysql.primary.tolerations", tolerations)
+		if err != nil {
+			return nil, err
+		}
+
 		imageRegistry := common.ThirdPartyContainerRepo(cfg.Config.Repository, common.DockerRegistryURL)
 
 		type EnvVar struct {
@@ -73,6 +82,7 @@ var Helm = common.CompositeHelmFunc(
 				FileValues: []string{
 					primaryAffinityTemplate,
 					extraEnvVarsTemplate,
+					tolerationsTemplate,
 				},
 			},
 		}, nil

install/installer/pkg/components/database/incluster/helm.go

@@ -65,12 +65,25 @@ var Helm = common.CompositeHelmFunc(
 			)
 		}
 
+		tolerations, err := helm.WithTolerationWorkspaceComponentNotReadyYaml(cfg)
+		if err != nil {
+			return nil, err
+		}
+		tolerationsTemplate, err := helm.KeyFileValue("docker-registry.tolerations", tolerations)
+		if err != nil {
+			return nil, err
+		}
+
 		registryValues = append(registryValues, helm.KeyValue("docker-registry.persistence.enabled", enablePersistence))
 
 		return &common.HelmConfig{
 			Enabled: inCluster,
 			Values: &values.Options{
 				Values: registryValues,
+				// This is too complex to be sent as a string
+				FileValues: []string{
+					tolerationsTemplate,
+				},
 			},
 		}, nil
 	}),

install/installer/pkg/components/docker-registry/helm.go

@@ -120,7 +120,8 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 						},
 							*common.KubeRBACProxyContainerWithConfig(ctx),
 						},
-						Volumes: volumes,
+						Volumes:     volumes,
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},

install/installer/pkg/components/ide-metrics/deployment.go

@@ -82,6 +82,7 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								TimeoutSeconds:   1,
 							},
 						}},
+						Tolerations: common.WithTolerationWorkspaceComponentNotReady(ctx),
 					},
 				},
 			},