[server] Introduces ReadinessProbe

components/server/src/container-module.ts

@@ -71,6 +71,7 @@ import { WebhookEventGarbageCollector } from "./jobs/webhook-gc";
 import { WorkspaceGarbageCollector } from "./jobs/workspace-gc";
 import { LinkedInService } from "./linkedin-service";
 import { LivenessController } from "./liveness/liveness-controller";
+import { ReadinessController } from "./liveness/readiness-controller";
 import { RedisSubscriber } from "./messaging/redis-subscriber";
 import { MonitoringEndpointsApp } from "./monitoring-endpoints";
 import { OAuthController } from "./oauth-server/oauth-controller";
@@ -135,6 +136,7 @@ import { AnalyticsController } from "./analytics-controller";
 import { InstallationAdminCleanup } from "./jobs/installation-admin-cleanup";
 import { AuditLogService } from "./audit/AuditLogService";
 import { AuditLogGarbageCollectorJob } from "./jobs/auditlog-gc";
+import { ProbesApp } from "./liveness/probes";
 
 export const productionContainerModule = new ContainerModule(
     (bind, unbind, isBound, rebind, unbindAsync, onActivation, onDeactivation) => {
@@ -240,7 +242,10 @@ export const productionContainerModule = new ContainerModule(
         bind(IWorkspaceManagerClientCallMetrics).toService(IClientCallMetrics);
 
         bind(WorkspaceDownloadService).toSelf().inSingletonScope();
+
+        bind(ProbesApp).toSelf().inSingletonScope();
         bind(LivenessController).toSelf().inSingletonScope();
+        bind(ReadinessController).toSelf().inSingletonScope();
 
         bind(OneTimeSecretServer).toSelf().inSingletonScope();
 

components/server/src/liveness/probes.ts

@@ -0,0 +1,25 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import express from "express";
+import { inject, injectable } from "inversify";
+import { LivenessController } from "./liveness-controller";
+import { ReadinessController } from "./readiness-controller";
+
+@injectable()
+export class ProbesApp {
+    constructor(
+        @inject(LivenessController) protected readonly livenessController: LivenessController,
+        @inject(ReadinessController) protected readonly readinessController: ReadinessController,
+    ) {}
+
+    public create(): express.Application {
+        const probesApp = express();
+        probesApp.use("/live", this.livenessController.apiRouter);
+        probesApp.use("/ready", this.readinessController.apiRouter);
+        return probesApp;
+    }
+}

components/server/src/liveness/readiness-controller.ts

@@ -0,0 +1,116 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { injectable, inject } from "inversify";
+import express from "express";
+import { TypeORM } from "@gitpod/gitpod-db/lib";
+import { SpiceDBClientProvider } from "../authorization/spicedb";
+import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
+import { ReadSchemaRequest } from "@authzed/authzed-node/dist/src/v1";
+import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
+import { Redis } from "ioredis";
+
+@injectable()
+export class ReadinessController {
+    @inject(TypeORM) protected readonly typeOrm: TypeORM;
+    @inject(SpiceDBClientProvider) protected readonly spiceDBClientProvider: SpiceDBClientProvider;
+    @inject(Redis) protected readonly redis: Redis;
+
+    get apiRouter(): express.Router {
+        const router = express.Router();
+        this.addReadinessHandler(router);
+        return router;
+    }
+
+    protected addReadinessHandler(router: express.Router) {
+        router.get("/", async (_, res) => {
+            try {
+                // Check feature flag first
+                const readinessProbeEnabled = await getExperimentsClientForBackend().getValueAsync(
+                    "server_readiness_probe",
+                    true, // Default to readiness probe, skip if false
+                    {},
+                );
+
+                if (!readinessProbeEnabled) {
+                    log.debug("Readiness check skipped due to feature flag");
+                    res.status(200);
+                    return;
+                }
+
+                // Check database connection
+                const dbConnection = await this.checkDatabaseConnection();
+                if (!dbConnection) {
+                    log.warn("Readiness check failed: Database connection failed");
+                    res.status(503).send("Database connection failed");
+                    return;
+                }
+
+                // Check SpiceDB connection
+                const spiceDBConnection = await this.checkSpiceDBConnection();
+                if (!spiceDBConnection) {
+                    log.warn("Readiness check failed: SpiceDB connection failed");
+                    res.status(503).send("SpiceDB connection failed");
+                    return;
+                }
+
+                // Check Redis connection
+                const redisConnection = await this.checkRedisConnection();
+                if (!redisConnection) {
+                    log.warn("Readiness check failed: Redis connection failed");
+                    res.status(503).send("Redis connection failed");
+                    return;
+                }
+
+                // All connections are good
+                res.status(200).send("Ready");
+            } catch (error) {
+                log.error("Readiness check failed", error);
+                res.status(503).send("Readiness check failed");
+            }
+        });
+    }
+
+    private async checkDatabaseConnection(): Promise<boolean> {
+        try {
+            const connection = await this.typeOrm.getConnection();
+            // Simple query to verify connection is working
+            await connection.query("SELECT 1");
+            return true;
+        } catch (error) {
+            log.error("Database connection check failed", error);
+            return false;
+        }
+    }
+
+    private async checkSpiceDBConnection(): Promise<boolean> {
+        try {
+            const client = this.spiceDBClientProvider.getClient();
+
+            // Send a request, to verify that the connection works
+            const req = ReadSchemaRequest.create({});
+            const response = await client.readSchema(req);
+            log.debug("SpiceDB connection check successful", { schemaLength: response.schemaText.length });
+
+            return true;
+        } catch (error) {
+            log.error("SpiceDB connection check failed", error);
+            return false;
+        }
+    }
+
+    private async checkRedisConnection(): Promise<boolean> {
+        try {
+            // Simple PING command to verify connection is working
+            const result = await this.redis.ping();
+            log.debug("Redis connection check successful", { result });
+            return result === "PONG";
+        } catch (error) {
+            log.error("Redis connection check failed", error);
+            return false;
+        }
+    }
+}

components/server/src/server.ts

@@ -36,7 +36,6 @@ import { NewsletterSubscriptionController } from "./user/newsletter-subscription
 import { Config } from "./config";
 import { DebugApp } from "@gitpod/gitpod-protocol/lib/util/debug-app";
 import { WsConnectionHandler } from "./express/ws-connection-handler";
-import { LivenessController } from "./liveness/liveness-controller";
 import { IamSessionApp } from "./iam/iam-session-app";
 import { API } from "./api/server";
 import { GithubApp } from "./prebuilds/github-app";
@@ -53,6 +52,11 @@ import {
 } from "./workspace/headless-log-service";
 import { runWithRequestContext } from "./util/request-context";
 import { AnalyticsController } from "./analytics-controller";
+import { ProbesApp as ProbesAppProvider } from "./liveness/probes";
+
+const MONITORING_PORT = 9500;
+const IAM_SESSION_PORT = 9876;
+const PROBES_PORT = 9400;
 
 @injectable()
 export class Server {
@@ -65,6 +69,8 @@ export class Server {
     protected privateApiServer?: http.Server;
 
     protected readonly eventEmitter = new EventEmitter();
+    protected probesApp: express.Application;
+    protected probesServer?: http.Server;
     protected app?: express.Application;
     protected httpServer?: http.Server;
     protected monitoringApp?: express.Application;
@@ -79,7 +85,6 @@ export class Server {
         @inject(UserController) private readonly userController: UserController,
         @inject(WebsocketConnectionManager) private readonly websocketConnectionHandler: WebsocketConnectionManager,
         @inject(WorkspaceDownloadService) private readonly workspaceDownloadService: WorkspaceDownloadService,
-        @inject(LivenessController) private readonly livenessController: LivenessController,
         @inject(MonitoringEndpointsApp) private readonly monitoringEndpointsApp: MonitoringEndpointsApp,
         @inject(CodeSyncService) private readonly codeSyncService: CodeSyncService,
         @inject(HeadlessLogController) private readonly headlessLogController: HeadlessLogController,
@@ -100,6 +105,7 @@ export class Server {
         @inject(API) private readonly api: API,
         @inject(RedisSubscriber) private readonly redisSubscriber: RedisSubscriber,
         @inject(AnalyticsController) private readonly analyticsController: AnalyticsController,
+        @inject(ProbesAppProvider) private readonly probesAppProvider: ProbesAppProvider,
     ) {}
 
     public async init(app: express.Application) {
@@ -113,6 +119,9 @@ export class Server {
         await this.typeOrm.connect();
         log.info("connected to DB");
 
+        // probes
+        this.probesApp = this.probesAppProvider.create();
+
         // metrics
         app.use((req: express.Request, res: express.Response, next: express.NextFunction) => {
             const startTime = Date.now();
@@ -319,7 +328,6 @@ export class Server {
         // Authorization: none
         app.use(this.oneTimeSecretServer.apiRouter);
         app.use(this.newsletterSubscriptionController.apiRouter);
-        app.use("/live", this.livenessController.apiRouter);
         app.use("/version", (req: express.Request, res: express.Response, next: express.NextFunction) => {
             res.send(this.config.version);
         });
@@ -351,22 +359,27 @@ export class Server {
             throw new Error("server cannot start, not initialized");
         }
 
+        const probeServer = this.probesApp.listen(PROBES_PORT, () => {
+            log.info(`probes server listening on port: ${(<AddressInfo>probeServer.address()).port}`);
+        });
+        this.probesServer = probeServer;
+
         const httpServer = this.app.listen(port, () => {
             this.eventEmitter.emit(Server.EVENT_ON_START, httpServer);
             log.info(`server listening on port: ${(<AddressInfo>httpServer.address()).port}`);
         });
         this.httpServer = httpServer;
 
         if (this.monitoringApp) {
-            this.monitoringHttpServer = this.monitoringApp.listen(9500, "localhost", () => {
+            this.monitoringHttpServer = this.monitoringApp.listen(MONITORING_PORT, "localhost", () => {
                 log.info(
                     `monitoring app listening on port: ${(<AddressInfo>this.monitoringHttpServer!.address()).port}`,
                 );
             });
         }
 
         if (this.iamSessionApp) {
-            this.iamSessionAppServer = this.iamSessionApp.listen(9876, () => {
+            this.iamSessionAppServer = this.iamSessionApp.listen(IAM_SESSION_PORT, () => {
                 log.info(
                     `IAM session server listening on port: ${(<AddressInfo>this.iamSessionAppServer!.address()).port}`,
                 );
@@ -406,6 +419,7 @@ export class Server {
             race(this.stopServer(this.httpServer), "stop httpserver"),
             race(this.stopServer(this.privateApiServer), "stop private api server"),
             race(this.stopServer(this.publicApiServer), "stop public api server"),
+            race(this.stopServer(this.probesServer), "stop probe server"),
             race((async () => this.disposables.dispose())(), "dispose disposables"),
         ]);
 

install/installer/pkg/components/server/constants.go

@@ -25,6 +25,8 @@ const (
 	DebugNodePortName                      = "debugnode"
 	ServicePort                            = 3000
 	personalAccessTokenSigningKeyMountPath = "/secrets/personal-access-token-signing-key"
+	ProbesPort                             = 9400
+	ProbesPortName                         = "probes"
 
 	AdminCredentialsSecretName      = "admin-credentials"
 	AdminCredentialsSecretMountPath = "/credentials/admin"

install/installer/pkg/components/server/deployment.go

@@ -363,43 +363,69 @@ func deployment(ctx *common.RenderContext) ([]runtime.Object, error) {
 										Path: "/live",
 										Port: intstr.IntOrString{
 											Type:   intstr.Int,
-											IntVal: ContainerPort,
+											IntVal: ProbesPort,
 										},
 									},
 								},
 								InitialDelaySeconds: 120,
 								PeriodSeconds:       10,
 								FailureThreshold:    6,
 							},
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Path: "/ready",
+										Port: intstr.IntOrString{
+											Type:   intstr.Int,
+											IntVal: ProbesPort,
+										},
+									},
+								},
+								InitialDelaySeconds: 5,
+								PeriodSeconds:       10,
+								FailureThreshold:    12, // try for 120 seconds, then re-start the container
+							},
 							SecurityContext: &corev1.SecurityContext{
 								Privileged:               pointer.Bool(false),
 								AllowPrivilegeEscalation: pointer.Bool(false),
 							},
-							Ports: []corev1.ContainerPort{{
-								Name:          ContainerPortName,
-								ContainerPort: ContainerPort,
-							}, {
-								Name:          baseserver.BuiltinMetricsPortName,
-								ContainerPort: baseserver.BuiltinMetricsPort,
-							}, {
-								Name:          InstallationAdminName,
-								ContainerPort: InstallationAdminPort,
-							}, {
-								Name:          IAMSessionPortName,
-								ContainerPort: IAMSessionPort,
-							}, {
-								Name:          DebugPortName,
-								ContainerPort: baseserver.BuiltinDebugPort,
-							}, {
-								Name:          DebugNodePortName,
-								ContainerPort: common.DebugNodePort,
-							}, {
-								Name:          GRPCAPIName,
-								ContainerPort: GRPCAPIPort,
-							}, {
-								Name:          PublicAPIName,
-								ContainerPort: PublicAPIPort,
-							},
+							Ports: []corev1.ContainerPort{
+								{
+									Name:          ContainerPortName,
+									ContainerPort: ContainerPort,
+								},
+								{
+									Name:          baseserver.BuiltinMetricsPortName,
+									ContainerPort: baseserver.BuiltinMetricsPort,
+								},
+								{
+									Name:          InstallationAdminName,
+									ContainerPort: InstallationAdminPort,
+								},
+								{
+									Name:          IAMSessionPortName,
+									ContainerPort: IAMSessionPort,
+								},
+								{
+									Name:          DebugPortName,
+									ContainerPort: baseserver.BuiltinDebugPort,
+								},
+								{
+									Name:          DebugNodePortName,
+									ContainerPort: common.DebugNodePort,
+								},
+								{
+									Name:          GRPCAPIName,
+									ContainerPort: GRPCAPIPort,
+								},
+								{
+									Name:          PublicAPIName,
+									ContainerPort: PublicAPIPort,
+								},
+								{
+									Name:          ProbesPortName,
+									ContainerPort: ProbesPort,
+								},
 							},
 							// todo(sje): do we need to cater for serverContainer.env from values.yaml?
 							Env: common.CustomizeEnvvar(ctx, Component, env),

memory-bank/.clinerules

@@ -26,6 +26,15 @@ This file captures important patterns, preferences, and project intelligence tha
 
 ## Development Workflow
 
+- **Feature Development Process**:
+  - Follow the Product Requirements Document (PRD) workflow documented in systemPatterns.md under "Development Workflows"
+  - Create PRD documents in the `prd/` directory with naming convention `NNN-featurename.md`
+  - Include standard sections: Overview, Background, Requirements, Implementation Details, Testing, Deployment Considerations, Future Improvements
+  - Use Plan Mode for requirements gathering and planning
+  - Use Act Mode for implementation and documentation updates
+  - Always update memory bank with new knowledge gained during implementation
+  - Reference the PRD in commit messages and documentation
+
 - **Build Approaches**:
   - **In-tree builds** (primary for local development):
     - TypeScript components: Use `yarn` commands defined in package.json