[server] Introduces ReadinessProbe

components/content-service/content-service.code-workspace

@@ -43,7 +43,6 @@
       },
       "go.lintTool": "golangci-lint",
       "gopls": {
-         "allowModfileModifications": true
       }
    }
 }

components/image-builder-mk3/image-builder.code-workspace

@@ -45,7 +45,6 @@
       },
       "go.lintTool": "golangci-lint",
       "gopls": {
-         "allowModfileModifications": true
       }
    }
 }

components/server/src/container-module.ts

@@ -71,6 +71,7 @@ import { WebhookEventGarbageCollector } from "./jobs/webhook-gc";
 import { WorkspaceGarbageCollector } from "./jobs/workspace-gc";
 import { LinkedInService } from "./linkedin-service";
 import { LivenessController } from "./liveness/liveness-controller";
+import { ReadinessController } from "./liveness/readiness-controller";
 import { RedisSubscriber } from "./messaging/redis-subscriber";
 import { MonitoringEndpointsApp } from "./monitoring-endpoints";
 import { OAuthController } from "./oauth-server/oauth-controller";
@@ -135,6 +136,7 @@ import { AnalyticsController } from "./analytics-controller";
 import { InstallationAdminCleanup } from "./jobs/installation-admin-cleanup";
 import { AuditLogService } from "./audit/AuditLogService";
 import { AuditLogGarbageCollectorJob } from "./jobs/auditlog-gc";
+import { ProbesApp } from "./liveness/probes";
 
 export const productionContainerModule = new ContainerModule(
     (bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {
@@ -240,7 +242,10 @@ export const productionContainerModule = new ContainerModule(
         bind(IWorkspaceManagerClientCallMetrics).toService(IClientCallMetrics);
 
         bind(WorkspaceDownloadService).toSelf().inSingletonScope();
+
+        bind(ProbesApp).toSelf().inSingletonScope();
         bind(LivenessController).toSelf().inSingletonScope();
+        bind(ReadinessController).toSelf().inSingletonScope();
 
         bind(OneTimeSecretServer).toSelf().inSingletonScope();
 

components/server/src/liveness/probes.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import express from "express";
+import { inject, injectable } from "inversify";
+import { LivenessController } from "./liveness-controller";
+import { ReadinessController } from "./readiness-controller";
+
+@injectable()
+export class ProbesApp {
+    constructor(
+        @inject(LivenessController) protected readonly livenessController: LivenessController,
+        @inject(ReadinessController) protected readonly readinessController: ReadinessController,
+    ) {}
+
+    public create(): express.Application {
+        const probesApp = express();
+        probesApp.use("/live", this.livenessController.apiRouter);
+        probesApp.use("/ready", this.readinessController.apiRouter);
+        return probesApp;
+    }
+}

components/server/src/liveness/readiness-controller.ts

@@ -0,0 +1,116 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { injectable, inject } from "inversify";
+import express from "express";
+import { TypeORM } from "@gitpod/gitpod-db/lib";
+import { SpiceDBClientProvider } from "../authorization/spicedb";
+import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { v1 } from "@authzed/authzed-node";
+import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
+import { Redis } from "ioredis";
+
+@injectable()
+export class ReadinessController {
+    @inject(TypeORM) protected readonly typeOrm: TypeORM;
+    @inject(SpiceDBClientProvider) protected readonly spiceDBClientProvider: SpiceDBClientProvider;
+    @inject(Redis) protected readonly redis: Redis;
+
+    get apiRouter(): express.Router {
+        const router = express.Router();
+        this.addReadinessHandler(router);
+        return router;
+    }
+
+    protected addReadinessHandler(router: express.Router) {
+        router.get("/", async (_, res) => {
+            try {
+                // Check feature flag first
+                const readinessProbeEnabled = await getExperimentsClientForBackend().getValueAsync(
+                    "server_readiness_probe",
+                    true, // Default to readiness probe, skip if false
+                    {},
+                );
+
+                if (!readinessProbeEnabled) {
+                    log.debug("Readiness check skipped due to feature flag");
+                    res.status(200);
+                    return;
+                }
+
+                // Check database connection
+                const dbConnection = await this.checkDatabaseConnection();
+                if (!dbConnection) {
+                    log.warn("Readiness check failed: Database connection failed");
+                    res.status(503).send("Database connection failed");
+                    return;
+                }
+
+                // Check SpiceDB connection
+                const spiceDBConnection = await this.checkSpiceDBConnection();
+                if (!spiceDBConnection) {
+                    log.warn("Readiness check failed: SpiceDB connection failed");
+                    res.status(503).send("SpiceDB connection failed");
+                    return;
+                }
+
+                // Check Redis connection
+                const redisConnection = await this.checkRedisConnection();
+                if (!redisConnection) {
+                    log.warn("Readiness check failed: Redis connection failed");
+                    res.status(503).send("Redis connection failed");
+                    return;
+                }
+
+                // All connections are good
+                res.status(200).send("Ready");
+            } catch (error) {
+                log.error("Readiness check failed", error);
+                res.status(503).send("Readiness check failed");
+            }
+        });
+    }
+
+    private async checkDatabaseConnection(): Promise<boolean> {
+        try {
+            const connection = await this.typeOrm.getConnection();
+            // Simple query to verify connection is working
+            await connection.query("SELECT 1");
+            return true;
+        } catch (error) {
+            log.error("Database connection check failed", error);
+            return false;
+        }
+    }
+
+    private async checkSpiceDBConnection(): Promise<boolean> {
+        try {
+            const client = this.spiceDBClientProvider.getClient();
+
+            // Send a request, to verify that the connection works
+            const req = v1.ReadSchemaRequest.create({});
+            const response = await client.readSchema(req);
+            log.debug("SpiceDB connection check successful", { schemaLength: response.schemaText.length });
+
+            return true;
+        } catch (error) {
+            log.error("SpiceDB connection check failed", error);
+            return false;
+        }
+    }
+
+    private async checkRedisConnection(): Promise<boolean> {
+        try {
+            // Simple PING command to verify connection is working
+            const result = await this.redis.ping();
+            log.debug("Redis connection check successful", { result });
+            return result === "PONG";
+        } catch (error) {
+            log.error("Redis connection check failed", error);
+            return false;
+        }
+    }
+}

components/server/src/server.ts

@@ -36,7 +36,6 @@ import { NewsletterSubscriptionController } from "./user/newsletter-subscription
 import { Config } from "./config";
 import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app";
 import { WsConnectionHandler } from "./express/ws-connection-handler";
-import { LivenessController } from "./liveness/liveness-controller";
 import { IamSessionApp } from "./iam/iam-session-app";
 import { API } from "./api/server";
 import { GithubApp } from "./prebuilds/github-app";
@@ -53,6 +52,11 @@ import {
 } from "./workspace/headless-log-service";
 import { runWithRequestContext } from "./util/request-context";
 import { AnalyticsController } from "./analytics-controller";
+import { ProbesApp as ProbesAppProvider } from "./liveness/probes";
+
+const MONITORING_PORT = 9500;
+const IAM_SESSION_PORT = 9876;
+const PROBES_PORT = 9400;
 
 @injectable()
 export class Server {
@@ -65,6 +69,8 @@ export class Server {
     protected privateApiServer?: http.Server;
 
     protected readonly eventEmitter = new EventEmitter();
+    protected probesApp: express.Application;
+    protected probesServer?: http.Server;
     protected app?: express.Application;
     protected httpServer?: http.Server;
     protected monitoringApp?: express.Application;
@@ -79,7 +85,6 @@ export class Server {
         @inject(UserController) private readonly userController: UserController,
         @inject(WebsocketConnectionManager) private readonly websocketConnectionHandler: WebsocketConnectionManager,
         @inject(WorkspaceDownloadService) private readonly workspaceDownloadService: WorkspaceDownloadService,
-        @inject(LivenessController) private readonly livenessController: LivenessController,
         @inject(MonitoringEndpointsApp) private readonly monitoringEndpointsApp: MonitoringEndpointsApp,
         @inject(CodeSyncService) private readonly codeSyncService: CodeSyncService,
         @inject(HeadlessLogController) private readonly headlessLogController: HeadlessLogController,
@@ -100,6 +105,7 @@ export class Server {
         @inject(API) private readonly api: API,
         @inject(RedisSubscriber) private readonly redisSubscriber: RedisSubscriber,
         @inject(AnalyticsController) private readonly analyticsController: AnalyticsController,
+        @inject(ProbesAppProvider) private readonly probesAppProvider: ProbesAppProvider,
     ) {}
 
     public async init(app: express.Application) {
@@ -113,6 +119,9 @@ export class Server {
         await this.typeOrm.connect();
         log.info("connected to DB");
 
+        // probes
+        this.probesApp = this.probesAppProvider.create();
+
         // metrics
         app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
             const startTime = Date.now();
@@ -319,7 +328,6 @@ export class Server {
         // Authorization: none
         app.use(this.oneTimeSecretServer.apiRouter);
         app.use(this.newsletterSubscriptionController.apiRouter);
-        app.use("/live", this.livenessController.apiRouter);
         app.use("/version", (req: express.Request, res: express.Response, next: express.NextFunction) => {
             res.send(this.config.version);
         });
@@ -351,22 +359,27 @@ export class Server {
             throw new Error("server cannot start, not initialized");
         }
 
+        const probeServer = this.probesApp.listen(PROBES_PORT, () => {
+            log.info(`probes server listening on port: ${(<AddressInfo>probeServer.address()).port}`);
+        });
+        this.probesServer = probeServer;
+
         const httpServer = this.app.listen(port, () => {
             this.eventEmitter.emit(Server.EVENT_ON_START, httpServer);
             log.info(`server listening on port: ${(<AddressInfo>httpServer.address()).port}`);
         });
         this.httpServer = httpServer;
 
         if (this.monitoringApp) {
-            this.monitoringHttpServer = this.monitoringApp.listen(9500, "localhost", () => {
+            this.monitoringHttpServer = this.monitoringApp.listen(MONITORING_PORT, "localhost", () => {
                 log.info(
                     `monitoring app listening on port: ${(<AddressInfo>this.monitoringHttpServer!.address()).port}`,
                 );
             });
         }
 
         if (this.iamSessionApp) {
-            this.iamSessionAppServer = this.iamSessionApp.listen(9876, () => {
+            this.iamSessionAppServer = this.iamSessionApp.listen(IAM_SESSION_PORT, () => {
                 log.info(
                     `IAM session server listening on port: ${(<AddressInfo>this.iamSessionAppServer!.address()).port}`,
                 );
@@ -406,6 +419,7 @@ export class Server {
             race(this.stopServer(this.httpServer), "stop httpserver"),
             race(this.stopServer(this.privateApiServer), "stop private api server"),
             race(this.stopServer(this.publicApiServer), "stop public api server"),
+            race(this.stopServer(this.probesServer), "stop probe server"),
             race((async () => this.disposables.dispose())(), "dispose disposables"),
         ]);
 

components/ws-manager-mk2/ws-manager-mk2.code-workspace

@@ -46,7 +46,6 @@
        },
        "go.lintTool": "golangci-lint",
        "gopls": {
-          "allowModfileModifications": true
        }
     }
  }

gitpod-ws.code-workspace

@@ -162,7 +162,6 @@
         "go.lintTool": "golangci-lint",
         "go.lintFlags": ["-disable", "govet,errcheck,staticcheck", "--allow-parallel-runners", "--timeout", "15m"],
         "gopls": {
-            "allowModfileModifications": true
         },
         "prettier.configPath": "/workspace/gitpod/.prettierrc.json"
     }

install/installer/pkg/components/server/constants.go

@@ -25,6 +25,8 @@ const (
 	DebugNodePortName                      = "debugnode"
 	ServicePort                            = 3000
 	personalAccessTokenSigningKeyMountPath = "/secrets/personal-access-token-signing-key"
+	ProbesPort                             = 9400
+	ProbesPortName                         = "probes"
 
 	AdminCredentialsSecretName      = "admin-credentials"
 	AdminCredentialsSecretMountPath = "/credentials/admin"