[db] Introduce DBWorkspaceInstanceMetrics and persist all metrics from ws-manager-api into it

Tool: gitpod/catfood.gitpod.cloud

Author: geropl

components/gitpod-db/src/typeorm/entity/db-workspace-instance-metrics.ts

@@ -0,0 +1,20 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { Entity, Column, PrimaryColumn } from "typeorm";
+import { WorkspaceInstanceMetrics } from "@gitpod/gitpod-protocol";
+
+@Entity()
+export class DBWorkspaceInstanceMetrics {
+    @PrimaryColumn()
+    instanceId: string;
+
+    @Column("json", { nullable: true })
+    metrics?: WorkspaceInstanceMetrics;
+
+    @Column()
+    _lastModified: Date;
+}

components/gitpod-db/src/typeorm/migration/1739892121734-AddWorkspaceInstanceMetricsTable.ts

@@ -0,0 +1,23 @@
+/**
+ * Copyright (c) 2025 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+
+import { MigrationInterface, QueryRunner } from "typeorm";
+
+export class AddWorkspaceInstanceMetricsTable1739892121734 implements MigrationInterface {
+    public async up(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(`CREATE TABLE IF NOT EXISTS d_b_workspace_instance_metrics (
+            instanceId char(36) NOT NULL,
+            metrics JSON,
+            _lastModified timestamp(6) NOT NULL DEFAULT CURRENT_TIMESTAMP(6) ON UPDATE CURRENT_TIMESTAMP(6),
+            PRIMARY KEY (instanceId),
+            KEY ind_dbsync (_lastModified)
+        ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;`);
+    }
+
+    public async down(queryRunner: QueryRunner): Promise<any> {
+        await queryRunner.query(`DROP TABLE IF EXISTS d_b_workspace_instance_metrics`);
+    }
+}

components/gitpod-db/src/typeorm/workspace-db-impl.ts

@@ -19,6 +19,7 @@ import {
     WorkspaceAndInstance,
     WorkspaceInfo,
     WorkspaceInstance,
+    WorkspaceInstanceMetrics,
     WorkspaceInstanceUser,
     WorkspaceSession,
     WorkspaceType,
@@ -62,6 +63,7 @@ import { TypeORM } from "./typeorm";
 import { ApplicationError, ErrorCodes } from "@gitpod/gitpod-protocol/lib/messaging/error";
 import { DBProject } from "./entity/db-project";
 import { PrebuiltWorkspaceWithWorkspace } from "@gitpod/gitpod-protocol/src/protocol";
+import { DBWorkspaceInstanceMetrics } from "./entity/db-workspace-instance-metrics-db";
 
 type RawTo<T> = (instance: WorkspaceInstance, ws: Workspace) => T;
 interface OrderBy {
@@ -109,6 +111,10 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl<WorkspaceDB> imp
         );
     }
 
+    private async getWorkspaceInstanceMetricsRepo(): Promise<Repository<DBWorkspaceInstanceMetrics>> {
+        return (await this.getEntityManager()).getRepository<DBWorkspaceInstanceMetrics>(DBWorkspaceInstanceMetrics);
+    }
+
     public async connect(maxTries: number = 3, timeout: number = 2000): Promise<void> {
         let tries = 1;
         while (tries <= maxTries) {
@@ -1143,6 +1149,36 @@ export class TypeORMWorkspaceDBImpl extends TransactionalDBImpl<WorkspaceDB> imp
         const res = await query.getMany();
         return res.map((r) => r.info);
     }
+
+    async storeMetrics(instanceId: string, metrics: WorkspaceInstanceMetrics): Promise<WorkspaceInstanceMetrics> {
+        const repo = await this.getWorkspaceInstanceMetricsRepo();
+        const result = await repo.save({
+            instanceId,
+            metrics,
+        });
+        return result.metrics;
+    }
+
+    async getMetrics(instanceId: string): Promise<WorkspaceInstanceMetrics | undefined> {
+        const repo = await this.getWorkspaceInstanceMetricsRepo();
+        const dbMetrics = await repo.findOne({ where: { instanceId } });
+        return dbMetrics?.metrics;
+    }
+
+    async updateMetrics(
+        instanceId: string,
+        update: WorkspaceInstanceMetrics,
+        merge: (current: WorkspaceInstanceMetrics, update: WorkspaceInstanceMetrics) => WorkspaceInstanceMetrics,
+    ): Promise<WorkspaceInstanceMetrics> {
+        return await this.transaction(async (db) => {
+            const current = await db.getMetrics(instanceId);
+            if (!current) {
+                return await db.storeMetrics(instanceId, update);
+            }
+            const merged = merge(current, update);
+            return await db.storeMetrics(instanceId, merged);
+        });
+    }
 }
 
 type InstanceJoinResult = DBWorkspace & { instance: WorkspaceInstance };

components/gitpod-db/src/workspace-db.ts

@@ -23,6 +23,7 @@ import {
     WorkspaceSession,
     PrebuiltWorkspaceWithWorkspace,
     PrebuildWithStatus,
+    WorkspaceInstanceMetrics,
 } from "@gitpod/gitpod-protocol";
 
 export type MaybeWorkspace = Workspace | undefined;
@@ -196,4 +197,12 @@ export interface WorkspaceDB {
 
     storePrebuildInfo(prebuildInfo: PrebuildInfo): Promise<void>;
     findPrebuildInfos(prebuildIds: string[]): Promise<PrebuildInfo[]>;
+
+    storeMetrics(instanceId: string, metrics: WorkspaceInstanceMetrics): Promise<WorkspaceInstanceMetrics>;
+    getMetrics(instanceId: string): Promise<WorkspaceInstanceMetrics | undefined>;
+    updateMetrics(
+        instanceId: string,
+        update: WorkspaceInstanceMetrics,
+        merge: (current: WorkspaceInstanceMetrics, update: WorkspaceInstanceMetrics) => WorkspaceInstanceMetrics,
+    ): Promise<WorkspaceInstanceMetrics>;
 }

components/gitpod-protocol/src/workspace-instance.ts

@@ -342,4 +342,30 @@ export interface WorkspaceInstanceMetrics {
          */
         workspaceImageSize: number;
     }>;
+
+    /**
+     * Metrics about the workspace initializer
+     */
+    initializerMetrics?: InitializerMetrics;
+}
+
+export interface InitializerMetrics {
+    git?: InitializerMetric;
+    fileDownload?: InitializerMetric;
+    snapshot?: InitializerMetric;
+    backup?: InitializerMetric;
+    prebuild?: InitializerMetric;
+    composite?: InitializerMetric;
+}
+
+export interface InitializerMetric {
+    /**
+     * Duration in milliseconds
+     */
+    duration: number;
+
+    /**
+     * Size in bytes
+     */
+    size: number;
 }

components/ws-manager-bridge/src/bridge.ts

@@ -22,6 +22,8 @@ import {
     PortProtocol as WsManPortProtocol,
     DescribeClusterRequest,
     WorkspaceType,
+    InitializerMetrics,
+    InitializerMetric,
 } from "@gitpod/ws-manager/lib";
 import { scrubber, TrustedValue } from "@gitpod/gitpod-protocol/lib/util/scrubbing";
 import { WorkspaceDB } from "@gitpod/gitpod-db/lib/workspace-db";
@@ -38,6 +40,7 @@ import { performance } from "perf_hooks";
 import { WorkspaceInstanceController } from "./workspace-instance-controller";
 import { PrebuildUpdater } from "./prebuild-updater";
 import { RedisPublisher } from "@gitpod/gitpod-db/lib";
+import { merge } from "ts-deepmerge";
 
 export const WorkspaceManagerBridgeFactory = Symbol("WorkspaceManagerBridgeFactory");
 
@@ -332,11 +335,13 @@ export class WorkspaceManagerBridge implements Disposable {
             instance.status.podName = instance.status.podName || status.runtime?.podName;
             instance.status.nodeIp = instance.status.nodeIp || status.runtime?.nodeIp;
             instance.status.ownerToken = status.auth!.ownerToken;
+            // TODO(gpl): fade this our in favor of only using DBWorkspaceInstanceMetrics
             instance.status.metrics = {
                 image: {
                     totalSize: instance.status.metrics?.image?.totalSize || status.metadata.metrics?.image?.totalSize,
                     workspaceImageSize:
-                        instance.status.metrics?.image?.totalSize || status.metadata.metrics?.image?.workspaceImageSize,
+                        instance.status.metrics?.image?.workspaceImageSize ||
+                        status.metadata.metrics?.image?.workspaceImageSize,
                 },
             };
 
@@ -411,6 +416,14 @@ export class WorkspaceManagerBridge implements Disposable {
             // now notify all prebuild listeners about updates - and update DB if needed
             await this.prebuildUpdater.updatePrebuiltWorkspace({ span }, userId, status);
 
+            // store metrics
+            const instanceMetrics = mapInstanceMetrics(status);
+            if (instanceMetrics) {
+                await this.workspaceDB
+                    .trace(ctx)
+                    .updateMetrics(instance.id, instanceMetrics, mergeWorkspaceInstanceMetrics);
+            }
+
             // cleanup
             // important: call this after the DB update
             if (!!lifecycleHandler) {
@@ -501,3 +514,64 @@ function toWorkspaceType(type: WorkspaceType): protocol.WorkspaceType {
     }
     throw new Error("invalid WorkspaceType: " + type);
 }
+
+function mergeWorkspaceInstanceMetrics(
+    current: protocol.WorkspaceInstanceMetrics,
+    update: protocol.WorkspaceInstanceMetrics,
+): protocol.WorkspaceInstanceMetrics {
+    const merged = merge.withOptions({ mergeArrays: false, allowUndefinedOverrides: false }, current, update);
+    return merged;
+}
+
+function mapInstanceMetrics(status: WorkspaceStatus.AsObject): protocol.WorkspaceInstanceMetrics | undefined {
+    let result: protocol.WorkspaceInstanceMetrics | undefined = undefined;
+
+    if (status.metadata?.metrics?.image) {
+        result = result || {};
+        result.image = {
+            totalSize: status.metadata.metrics.image.totalSize,
+            workspaceImageSize: status.metadata.metrics.image.workspaceImageSize,
+        };
+    }
+    if (status.initializerMetrics) {
+        result = result || {};
+        result.initializerMetrics = mapInitializerMetrics(status.initializerMetrics);
+    }
+
+    return result;
+}
+
+function mapInitializerMetrics(metrics: InitializerMetrics.AsObject): protocol.InitializerMetrics {
+    const result: protocol.InitializerMetrics = {};
+    if (metrics.git) {
+        result.git = mapInitializerMetric(metrics.git);
+    }
+    if (metrics.fileDownload) {
+        result.fileDownload = mapInitializerMetric(metrics.fileDownload);
+    }
+    if (metrics.snapshot) {
+        result.snapshot = mapInitializerMetric(metrics.snapshot);
+    }
+    if (metrics.backup) {
+        result.backup = mapInitializerMetric(metrics.backup);
+    }
+    if (metrics.prebuild) {
+        result.prebuild = mapInitializerMetric(metrics.prebuild);
+    }
+    if (metrics.composite) {
+        result.composite = mapInitializerMetric(metrics.composite);
+    }
+
+    return result;
+}
+
+function mapInitializerMetric(metric: InitializerMetric.AsObject | undefined): protocol.InitializerMetric | undefined {
+    if (!metric || !metric.duration) {
+        return undefined;
+    }
+
+    return {
+        duration: metric.duration.seconds * 1000 + metric.duration.nanos / 1000000,
+        size: metric.size,
+    };
+}