[server] Introduce special /ready handler that only returns "false" during the shutdown phase

Tool: gitpod/catfood.gitpod.cloud

Author: geropl

components/server/src/container-module.ts

@@ -137,6 +137,7 @@ import { InstallationAdminCleanup } from "./jobs/installation-admin-cleanup";
 import { AuditLogService } from "./audit/AuditLogService";
 import { AuditLogGarbageCollectorJob } from "./jobs/auditlog-gc";
 import { ProbesApp } from "./liveness/probes";
+import { ReadinessController } from "./liveness/readiness-controller";
 
 export const productionContainerModule = new ContainerModule(
     (bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {
@@ -246,6 +247,7 @@ export const productionContainerModule = new ContainerModule(
         bind(ProbesApp).toSelf().inSingletonScope();
         bind(LivenessController).toSelf().inSingletonScope();
         bind(StartupController).toSelf().inSingletonScope();
+        bind(ReadinessController).toSelf().inSingletonScope();
 
         bind(OneTimeSecretServer).toSelf().inSingletonScope();
 

components/server/src/liveness/probes.ts

@@ -10,6 +10,7 @@ import { inject, injectable } from "inversify";
 import { LivenessController } from "./liveness-controller";
 import { StartupController } from "./startup-controller";
 import { AddressInfo } from "net";
+import { ReadinessController } from "./readiness-controller";
 
 @injectable()
 export class ProbesApp {
@@ -19,10 +20,12 @@ export class ProbesApp {
     constructor(
         @inject(LivenessController) protected readonly livenessController: LivenessController,
         @inject(StartupController) protected readonly startupController: StartupController,
+        @inject(ReadinessController) protected readonly readinessController: ReadinessController,
     ) {
         const probesApp = express();
         probesApp.use("/live", this.livenessController.apiRouter);
         probesApp.use("/startup", this.startupController.apiRouter);
+        probesApp.use("/ready", this.readinessController.apiRouter);
         this.app = probesApp;
     }
 
@@ -35,6 +38,10 @@ export class ProbesApp {
         });
     }
 
+    public notifyShutdown(): void {
+        this.readinessController.notifyShutdown();
+    }
+
     public async stop(): Promise<void> {
         this.httpServer?.close();
     }

components/server/src/liveness/readiness-controller.ts

@@ -0,0 +1,45 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { injectable } from "inversify";
+import express from "express";
+import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+
+/**
+ * ReadinessController is mimicking the behavior server had in the past: Behave as there is not ready probe - except during shutdown.
+ *
+ * Why? In Gitpod, our error strategy has always been "keep it local and retry", instead of "fail loud and have someone else handle it".
+ * As we don't want to change this now, we keep the same behavior for most of the services lifetime.
+ *
+ * Only during shutdown, we want to signal that the service is not ready anymore, to reduce error peaks.
+ */
+@injectable()
+export class ReadinessController {
+    private shutdown: boolean = false;
+
+    get apiRouter(): express.Router {
+        const router = express.Router();
+        this.addReadinessHandler(router);
+        return router;
+    }
+
+    public notifyShutdown(): void {
+        this.shutdown = true;
+    }
+
+    protected addReadinessHandler(router: express.Router) {
+        router.get("/", async (_, res) => {
+            if (this.shutdown) {
+                log.warn("Readiness check failed: Server is shutting down");
+                res.status(503).send("Server is shutting down");
+                return;
+            }
+
+            res.status(200).send("Ready");
+            log.debug("Readiness check successful");
+        });
+    }
+}

components/server/src/server.ts

@@ -387,6 +387,9 @@ export class Server {
     }
 
     public async stop() {
+        // mark as not-ready
+        this.probesApp.notifyShutdown();
+
         // run each stop with a timeout of 30s
         async function race(workLoad: Promise<any>, task: string, ms: number = 30 * 1000): Promise<void> {
             const before = Date.now();
@@ -413,10 +416,13 @@ export class Server {
             race(this.stopServer(this.httpServer), "stop httpserver"),
             race(this.stopServer(this.privateApiServer), "stop private api server"),
             race(this.stopServer(this.publicApiServer), "stop public api server"),
-            race(this.probesApp.stop(), "stop probe server"),
             race((async () => this.disposables.dispose())(), "dispose disposables"),
         ]);
 
+        this.probesApp.stop().catch(() => {
+            /* ignore */
+        });
+
         log.info("server stopped.");
     }
 

install/installer/pkg/components/server/deployment.go

@@ -387,6 +387,21 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 								PeriodSeconds:       10,
 								FailureThreshold:    18, // try for 180 seconds, then the Pod is restarted
 							},
+							// /ready will only return false on shutdown (SIGTERM), always true otherwise
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/ready",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: ProbesPort,
+										},
+									},
+								},
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       5,
+								FailureThreshold:    1, // mark as "not ready" as quick as possible after receiving SIGTERM
+							},
 							SecurityContext: &corev1.SecurityContext{
 								Privileged:               pointer.Bool(false),
 								AllowPrivilegeEscalation: pointer.Bool(false),