[ws-manager-mk2] update to support k8s 0.30 API

Without which, we cannot remove the gitpod finalizer, pods do not terminate, and as such, we do not scale down nodes

Tool: gitpod/catfood.gitpod.cloud

Author: geropl

components/ws-manager-mk2/controllers/create.go

@@ -21,6 +21,7 @@ import (
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+	"k8s.io/apimachinery/pkg/version"
 	"k8s.io/utils/pointer"
 
 	wsk8s "github.com/gitpod-io/gitpod/common-go/kubernetes"
@@ -62,6 +63,7 @@ type startWorkspaceContext struct {
 	IDEPort        int32             `json:"idePort"`
 	SupervisorPort int32             `json:"supervisorPort"`
 	Headless       bool              `json:"headless"`
+	ServerVersion  *version.Info     `json:"serverVersion"`
 }
 
 // createWorkspacePod creates the actual workspace pod based on the definite workspace pod and appropriate
@@ -278,12 +280,13 @@ func createDefiniteWorkspacePod(sctx *startWorkspaceContext) (*corev1.Pod, error
 		"prometheus.io/scrape": "true",
 		"prometheus.io/path":   "/metrics",
 		"prometheus.io/port":   strconv.Itoa(int(sctx.IDEPort)),
-		"container.apparmor.security.beta.kubernetes.io/workspace": "unconfined",
 		// prevent cluster-autoscaler from removing a node
 		// https://github.com/kubernetes/autoscaler/blob/master/cluster-autoscaler/FAQ.md#what-types-of-pods-can-prevent-ca-from-removing-a-node
 		"cluster-autoscaler.kubernetes.io/safe-to-evict": "false",
 	}
 
+	configureAppamor(sctx, annotations, workspaceContainer)
+
 	for k, v := range sctx.Workspace.Annotations {
 		annotations[k] = v
 	}
@@ -689,7 +692,7 @@ func createDefaultSecurityContext() (*corev1.SecurityContext, error) {
 	return res, nil
 }
 
-func newStartWorkspaceContext(ctx context.Context, cfg *config.Configuration, ws *workspacev1.Workspace) (res *startWorkspaceContext, err error) {
+func newStartWorkspaceContext(ctx context.Context, cfg *config.Configuration, ws *workspacev1.Workspace, serverVersion *version.Info) (res *startWorkspaceContext, err error) {
 	// we deliberately do not shadow ctx here as we need the original context later to extract the TraceID
 	span, _ := tracing.FromContext(ctx, "newStartWorkspaceContext")
 	defer tracing.FinishSpan(span, &err)
@@ -711,9 +714,25 @@ func newStartWorkspaceContext(ctx context.Context, cfg *config.Configuration, ws
 		IDEPort:        23000,
 		SupervisorPort: 22999,
 		Headless:       ws.IsHeadless(),
+		ServerVersion:  serverVersion,
 	}, nil
 }
 
+func configureAppamor(sctx *startWorkspaceContext, annotations map[string]string, workspaceContainer *corev1.Container) {
+	// pre K8s 1.30 we need to set the apparmor profile to unconfined as an annotation
+	if sctx.ServerVersion.Major <= "1" && sctx.ServerVersion.Minor <= "30" {
+		annotations["container.apparmor.security.beta.kubernetes.io/workspace"] = "unconfined"
+	} else {
+
+		// TODO: set AppArmorProfile field here, if the K8s minor version is >= 30
+		// Ref: https://pkg.go.dev/k8s.io/[email protected]/core/v1#SecurityContext
+		// and https://pkg.go.dev/k8s.io/[email protected]/core/v1#AppArmorProfile
+		// and https://pkg.go.dev/k8s.io/[email protected]/core/v1#AppArmorProfileType
+		//
+		// requires we update k8s libraries to 0.30.8
+	}
+}
+
 // validCookieChars contains all characters which may occur in an HTTP Cookie value (unicode \u0021 through \u007E),
 // without the characters , ; and / ... I did not find more details about permissible characters in RFC2965, so I took
 // this list of permissible chars from Wikipedia.

components/ws-manager-mk2/controllers/workspace_controller.go

@@ -18,6 +18,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apimachinery/pkg/version"
+	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -75,6 +77,8 @@ type WorkspaceReconciler struct {
 	metrics     *controllerMetrics
 	maintenance maintenance.Maintenance
 	Recorder    record.EventRecorder
+
+	kubeClient kubernetes.Interface
 }
 
 //+kubebuilder:rbac:groups=workspace.gitpod.io,resources=workspaces,verbs=get;list;watch;create;update;patch;delete
@@ -181,7 +185,8 @@ func (r *WorkspaceReconciler) actOnStatus(ctx context.Context, workspace *worksp
 		// if there isn't a workspace pod and we're not currently deleting this workspace,// create one.
 		switch {
 		case workspace.Status.PodStarts == 0 || workspace.Status.PodStarts-workspace.Status.PodRecreated < 1:
-			sctx, err := newStartWorkspaceContext(ctx, r.Config, workspace)
+			serverVersion := r.getServerVersion(ctx)
+			sctx, err := newStartWorkspaceContext(ctx, r.Config, workspace, serverVersion)
 			if err != nil {
 				log.Error(err, "unable to create startWorkspace context")
 				return ctrl.Result{Requeue: true}, err
@@ -627,6 +632,20 @@ func (r *WorkspaceReconciler) SetupWithManager(mgr ctrl.Manager) error {
 		Complete(r)
 }
 
+func (r *WorkspaceReconciler) getServerVersion(ctx context.Context) *version.Info {
+	log := log.FromContext(ctx)
+
+	serverVersion, err := r.kubeClient.Discovery().ServerVersion()
+	if err != nil {
+		log.Error(err, "cannot get server version! Assuming 1.30 going forward")
+		serverVersion = &version.Info{
+			Major: "1",
+			Minor: "30",
+		}
+	}
+	return serverVersion
+}
+
 func SetupIndexer(mgr ctrl.Manager) error {
 	var err error
 	var once sync.Once