[ws-scheduler] Make ghost workspaces more effective by integrating them with scheduler (#2552)

* [ws-manager] Do not delete ghost workspace on start

* [ws-scheduler] Enable asynchronous binding of pods

* [ws-scheduler] Introduce ghosts

 - remove a ghost before binding a regular workspace
 - make ghosts "invisible" to strategy

* [scheduler] Wait longer on ghost deletion to prevent OOM errors

* [scheduler] Make isRegularWorkspace -> makeGhostsInvisible explicit

* [scheduler] cancel ghost.Delete if it takes too long (5s)

* [ws-scheduler] Add tests for ghost-sepcific state computation

* [scheduler] Make sure ghost are only selected for deletion once

* [scheduler] delete ghosts: ctxDeleteTimeout > gracePeriod

* [scheduler] Don't bind terminated pods

* [scheduler] Make all non-ghost workspaces replace ghosts

* [scheduler] review comments

Author: geropl

chart/templates/ws-scheduler-clusterrole.yaml

@@ -24,6 +24,7 @@ rules:
     resources:
     - "pods"
     verbs:
+    - "delete"
     - "get"
     - "list"
     - "watch"

components/common-go/go.mod

@@ -18,6 +18,7 @@ require (
 	go.uber.org/atomic v1.4.0 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e
 	google.golang.org/grpc v1.34.0
+	k8s.io/api v0.20.1
 	k8s.io/apimachinery v0.20.1
 	k8s.io/client-go v0.0.0
 )

components/common-go/go.sum

@@ -484,6 +484,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg=
 honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
+k8s.io/api v0.20.1 h1:ud1c3W3YNzGd6ABJlbFfKXBKXO+1KdGfcgGGNgFR03E=
 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo=
 k8s.io/apimachinery v0.20.1 h1:LAhz8pKbgR8tUwn7boK+b2HZdt7MiTu2mkYtFMUjTRQ=
 k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU=

components/common-go/kubernetes/kubernetes.go

@@ -19,6 +19,7 @@ import (
 	"github.com/gitpod-io/gitpod/common-go/log"
 	"github.com/sirupsen/logrus"
 
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/util/flowcontrol"
 )
@@ -99,3 +100,31 @@ func (u *UnlimitedRateLimiter) QPS() float32 {
 func (u *UnlimitedRateLimiter) Wait(ctx context.Context) error {
 	return nil
 }
+
+func IsWorkspace(pod *corev1.Pod) bool {
+	val, ok := pod.ObjectMeta.Labels["component"]
+	return ok && val == "workspace"
+}
+
+func IsHeadlessWorkspace(pod *corev1.Pod) bool {
+	if !IsWorkspace(pod) {
+		return false
+	}
+
+	val, ok := pod.ObjectMeta.Labels["headless"]
+	return ok && val == "true"
+}
+
+func IsGhostWorkspace(pod *corev1.Pod) bool {
+	if !IsWorkspace(pod) {
+		return false
+	}
+
+	val, ok := pod.ObjectMeta.Labels[TypeLabel]
+	return ok && val == "ghost"
+}
+
+func IsNonGhostWorkspace(pod *corev1.Pod) bool {
+	return IsWorkspace(pod) &&
+		!IsGhostWorkspace(pod)
+}

components/ee/ws-scheduler/.gitignore

@@ -0,0 +1 @@
+/ws-scheduler

components/ee/ws-scheduler/cmd/test-cluster-scaleup.go

@@ -231,7 +231,7 @@ func buildCurrentState(clientSet *kubernetes.Clientset) (*scheduler.State, error
 	}
 
 	ramSafetyBuffer := res.MustParse("0Mi")
-	state := scheduler.ComputeState(potentialNodes, pods, nil, &ramSafetyBuffer)
+	state := scheduler.ComputeState(potentialNodes, pods, nil, &ramSafetyBuffer, false)
 	return state, nil
 }