[server] make gRPC clients viable in non-HTTP/2-compatible environments (#20565)

filiptronicek · geropl · web-flow · commit 9b574a902220 · 2025-01-30T07:00:19.000-05:00
* [server] make gRPC clients viable in non-HTTP/2-compatible environments

Tool: gitpod/catfood.gitpod.cloud

* Address review comments

Co-authored-by: Gero Posmyk-Leinemann &lt;gero@gitpod.io&gt;
Tool: gitpod/catfood.gitpod.cloud

---------

Co-authored-by: Gero Posmyk-Leinemann &lt;gero@gitpod.io&gt;
diff --git a/components/dashboard/src/service/public-api.ts b/components/dashboard/src/service/public-api.ts
@@ -10,11 +10,8 @@ import { CallOptions, Code, ConnectError, PromiseClient, createPromiseClient } f
 import { createConnectTransport } from "@connectrpc/connect-web";
 import { Disposable } from "@gitpod/gitpod-protocol";
 import { PublicAPIConverter } from "@gitpod/public-api-common/lib/public-api-converter";
-import { Project as ProtocolProject } from "@gitpod/gitpod-protocol/lib/teams-projects-protocol";
 import { HelloService } from "@gitpod/public-api/lib/gitpod/experimental/v1/dummy_connect";
 import { OIDCService } from "@gitpod/public-api/lib/gitpod/experimental/v1/oidc_connect";
-import { ProjectsService } from "@gitpod/public-api/lib/gitpod/experimental/v1/projects_connect";
-import { Project } from "@gitpod/public-api/lib/gitpod/experimental/v1/projects_pb";
 import { TokensService } from "@gitpod/public-api/lib/gitpod/experimental/v1/tokens_connect";
 import { OrganizationService } from "@gitpod/public-api/lib/gitpod/v1/organization_connect";
 import { WorkspaceService } from "@gitpod/public-api/lib/gitpod/v1/workspace_connect";
@@ -52,10 +49,6 @@ export const converter = new PublicAPIConverter();
 
 export const helloService = createServiceClient(HelloService);
 export const personalAccessTokensService = createPromiseClient(TokensService, transport);
-/**
- * @deprecated use configurationClient instead
- */
-export const projectsService = createPromiseClient(ProjectsService, transport);
 
 export const oidcService = createPromiseClient(OIDCService, transport);
 
@@ -68,7 +61,7 @@ export const organizationClient = createServiceClient(OrganizationService, {
     featureFlagSuffix: "organization",
 });
 
-// No jsonrcp client for the configuration service as it's only used in new UI of the dashboard
+// No jsonrpc client for the configuration service as it's only used in new UI of the dashboard
 export const configurationClient = createServiceClient(ConfigurationService);
 export const prebuildClient = createServiceClient(PrebuildService, {
     client: new JsonRpcPrebuildClient(),
@@ -110,56 +103,6 @@ export const installationClient = createServiceClient(InstallationService, {
     featureFlagSuffix: "installation",
 });
 
-export async function listAllProjects(opts: { orgId: string }): Promise<ProtocolProject[]> {
-    let pagination = {
-        page: 1,
-        pageSize: 100,
-    };
-
-    const response = await projectsService.listProjects({
-        teamId: opts.orgId,
-        pagination,
-    });
-    const results = response.projects;
-
-    while (results.length < response.totalResults) {
-        pagination = {
-            pageSize: 100,
-            page: 1 + pagination.page,
-        };
-        const response = await projectsService.listProjects({
-            teamId: opts.orgId,
-            pagination,
-        });
-        results.push(...response.projects);
-    }
-
-    return results.map(projectToProtocol);
-}
-
-export function projectToProtocol(project: Project): ProtocolProject {
-    return {
-        id: project.id,
-        name: project.name,
-        cloneUrl: project.cloneUrl,
-        creationTime: project.creationTime?.toDate().toISOString() || "",
-        teamId: project.teamId,
-        appInstallationId: "undefined",
-        settings: {
-            workspaceClasses: {
-                regular: project.settings?.workspace?.workspaceClass?.regular || "",
-            },
-            prebuilds: {
-                enable: project.settings?.prebuild?.enablePrebuilds,
-                branchStrategy: project.settings?.prebuild?.branchStrategy as any,
-                branchMatchingPattern: project.settings?.prebuild?.branchMatchingPattern,
-                prebuildInterval: project.settings?.prebuild?.prebuildInterval,
-                workspaceClass: project.settings?.prebuild?.workspaceClass,
-            },
-        },
-    };
-}
-
 let user: { id: string; email?: string } | undefined;
 export function updateUserForExperiments(newUser?: { id: string; email?: string }) {
     user = newUser;
@@ -176,10 +119,13 @@ function createServiceClient<T extends ServiceType>(
         get(grpcClient, prop) {
             const experimentsClient = getExperimentsClient();
             // TODO(ak) remove after migration
-            async function resolveClient(): Promise<PromiseClient<T>> {
+            async function resolveClient(preferJsonRpc?: boolean): Promise<PromiseClient<T>> {
                 if (!jsonRpcOptions) {
                     return grpcClient;
                 }
+                if (preferJsonRpc) {
+                    return jsonRpcOptions.client;
+                }
                 const featureFlags = [`dashboard_public_api_${jsonRpcOptions.featureFlagSuffix}_enabled`];
                 const resolvedFlags = await Promise.all(
                     featureFlags.map((ff) =>
@@ -241,7 +187,8 @@ function createServiceClient<T extends ServiceType>(
                 }
                 return (async function* () {
                     try {
-                        const client = await resolveClient();
+                        // for server streaming, we prefer jsonRPC
+                        const client = await resolveClient(true);
                         const generator = Reflect.apply(client[prop as any], client, args) as AsyncGenerator<any>;
                         for await (const item of generator) {
                             yield item;
diff --git a/components/dashboard/src/service/service.tsx b/components/dashboard/src/service/service.tsx
@@ -33,7 +33,7 @@ import { sendTrackEvent } from "../Analytics";
 export const gitpodHostUrl = new GitpodHostUrl(window.location.toString());
 
 function createGitpodService<C extends GitpodClient, S extends GitpodServer>() {
-    let host = gitpodHostUrl.asWebsocket().with({ pathname: GitpodServerPath }).withApi();
+    const host = gitpodHostUrl.asWebsocket().with({ pathname: GitpodServerPath }).withApi();
 
     const connectionProvider = new WebSocketConnectionProvider();
     instrumentWebSocketConnection(connectionProvider);
diff --git a/components/gitpod-db/src/redis/publisher.ts b/components/gitpod-db/src/redis/publisher.ts
@@ -23,13 +23,13 @@ export class RedisPublisher {
     constructor(@inject(Redis) private readonly redis: Redis) {}
 
     async publishPrebuildUpdate(update: RedisPrebuildUpdate): Promise<void> {
-        log.debug("[redis] Publish prebuild udpate invoked.");
+        log.debug("[redis] Publish prebuild update invoked.");
 
         let err: Error | undefined;
         try {
             const serialized = JSON.stringify(update);
             await this.redis.publish(PrebuildUpdatesChannel, serialized);
-            log.debug("[redis] Succesfully published prebuild update.", update);
+            log.debug("[redis] Successfully published prebuild update.", update);
         } catch (e) {
             err = e;
             log.error("[redis] Failed to publish prebuild update.", e, update);
@@ -43,7 +43,7 @@ export class RedisPublisher {
         try {
             const serialized = JSON.stringify(update);
             await this.redis.publish(WorkspaceInstanceUpdatesChannel, serialized);
-            log.debug("[redis] Succesfully published instance update.", update);
+            log.debug("[redis] Successfully published instance update.", update);
         } catch (e) {
             err = e;
             log.error("[redis] Failed to publish instance update.", e, update);
@@ -53,13 +53,13 @@ export class RedisPublisher {
     }
 
     async publishHeadlessUpdate(update: RedisHeadlessUpdate): Promise<void> {
-        log.debug("[redis] Publish headless udpate invoked.");
+        log.debug("[redis] Publish headless update invoked.");
 
         let err: Error | undefined;
         try {
             const serialized = JSON.stringify(update);
             await this.redis.publish(HeadlessUpdatesChannel, serialized);
-            log.debug("[redis] Succesfully published headless update.", update);
+            log.debug("[redis] Successfully published headless update.", update);
         } catch (e) {
             err = e;
             log.error("[redis] Failed to publish headless update.", e, update);
diff --git a/components/gitpod-protocol/src/redis.ts b/components/gitpod-protocol/src/redis.ts
@@ -22,6 +22,7 @@ export type RedisPrebuildUpdate = {
     prebuildID: string;
     workspaceID: string;
     projectID: string;
+    organizationID?: string;
 };
 
 export type RedisHeadlessUpdate = {
diff --git a/components/server/src/messaging/redis-subscriber.ts b/components/server/src/messaging/redis-subscriber.ts
@@ -67,7 +67,7 @@ export class RedisSubscriber {
                 let err: Error | undefined;
                 try {
                     await this.onMessage(channel, message);
-                    log.debug("[redis] Succesfully handled update", { channel, message });
+                    log.debug("[redis] Successfully handled update", { channel, message });
                 } catch (e) {
                     err = e;
                     log.error("[redis] Failed to handle message from Pub/Sub", e, { channel, message });
@@ -132,7 +132,10 @@ export class RedisSubscriber {
             return;
         }
 
-        const listeners = this.prebuildUpdateListeners.get(update.projectID) || [];
+        const listeners = this.prebuildUpdateListeners.get(update.projectID) ?? [];
+        if (update.organizationID) {
+            listeners.push(...(this.prebuildUpdateListeners.get(update.organizationID) ?? []));
+        }
         if (listeners.length === 0) {
             return;
         }
@@ -182,10 +185,14 @@ export class RedisSubscriber {
         this.disposables.dispose();
     }
 
-    listenForPrebuildUpdates(projectId: string, listener: PrebuildUpdateListener): Disposable {
+    listenForProjectPrebuildUpdates(projectId: string, listener: PrebuildUpdateListener): Disposable {
         return this.doRegister(projectId, listener, this.prebuildUpdateListeners, "prebuild");
     }
 
+    listenForOrganizationPrebuildUpdates(organizationId: string, listener: PrebuildUpdateListener): Disposable {
+        return this.doRegister(organizationId, listener, this.prebuildUpdateListeners, "prebuild");
+    }
+
     listenForPrebuildUpdatableEvents(listener: HeadlessWorkspaceEventListener): Disposable {
         // we're being cheap here in re-using a map where it just needs to be a plain array.
         return this.doRegister(UNDEFINED_KEY, listener, this.headlessWorkspaceEventListeners, "prebuild-updatable");
diff --git a/components/server/src/prebuilds/prebuild-manager.ts b/components/server/src/prebuilds/prebuild-manager.ts
@@ -100,7 +100,7 @@ export class PrebuildManager {
         await this.auth.checkPermissionOnProject(userId, "read_prebuild", configurationId);
         return generateAsyncGenerator<PrebuildWithStatus>((sink) => {
             try {
-                const toDispose = this.subscriber.listenForPrebuildUpdates(configurationId, (_ctx, prebuild) => {
+                const toDispose = this.subscriber.listenForProjectPrebuildUpdates(configurationId, (_ctx, prebuild) => {
                     sink.push(prebuild);
                 });
                 return () => {
diff --git a/components/server/src/workspace/gitpod-server-impl.ts b/components/server/src/workspace/gitpod-server-impl.ts
@@ -236,30 +236,17 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
         log.debug({ userId: this.userID }, "initializeClient");
 
         this.listenForWorkspaceInstanceUpdates();
-        this.listenForPrebuildUpdates().catch((err) => log.error("error registering for prebuild updates", err));
+        this.listenForPrebuildUpdates(connectionCtx).catch((err) =>
+            log.error("error registering for prebuild updates", err),
+        );
     }
 
-    private async listenForPrebuildUpdates() {
-        if (!this.client) {
-            return;
-        }
-
-        // todo(ft) disable registering for all updates from all projects by default and only listen to updates when the client is explicity interested in them
-        const disableWebsocketPrebuildUpdates = await getExperimentsClientForBackend().getValueAsync(
-            "disableWebsocketPrebuildUpdates",
-            false,
-            {
-                gitpodHost: this.config.hostUrl.url.host,
-            },
-        );
-        if (disableWebsocketPrebuildUpdates) {
-            log.info("ws prebuild updates disabled by feature flag");
+    private async listenForPrebuildUpdates(ctx?: TraceContext) {
+        const userId = this.userID;
+        if (!this.client || !userId) {
             return;
         }
 
-        // 'registering for prebuild updates for all projects this user has access to
-        const projects = await this.getAccessibleProjects();
-
         const handler = (ctx: TraceContext, update: PrebuildWithStatus) =>
             TraceContext.withSpan(
                 "forwardPrebuildUpdateToClient",
@@ -273,38 +260,30 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
             );
 
         if (!this.disposables.disposed) {
-            for (const project of projects) {
-                this.disposables.push(this.subscriber.listenForPrebuildUpdates(project.id, handler));
-            }
-        }
-
-        // TODO(at) we need to keep the list of accessible project up to date
-    }
-
-    private async getAccessibleProjects() {
-        const userId = this.userID;
-        if (!userId) {
-            return [];
+            await runWithRequestContext(
+                {
+                    requestKind: "gitpod-server-impl-listener",
+                    requestMethod: "listenForPrebuildUpdates",
+                    signal: new AbortController().signal,
+                    subjectId: SubjectId.fromUserId(userId),
+                },
+                async () => {
+                    const organizations = await this.getTeams(ctx ?? {});
+                    for (const organization of organizations) {
+                        const hasPermission = await this.auth.hasPermissionOnOrganization(
+                            userId,
+                            "read_prebuild",
+                            organization.id,
+                        );
+                        if (hasPermission) {
+                            this.disposables.push(
+                                this.subscriber.listenForOrganizationPrebuildUpdates(organization.id, handler),
+                            );
+                        }
+                    }
+                },
+            );
         }
-
-        // update all project this user has access to
-        // gpl: This call to runWithRequestContext is not nice, but it's only there to please the old impl for a limited time, so it's fine.
-        return runWithRequestContext(
-            {
-                requestKind: "gitpod-server-impl-listener",
-                requestMethod: "getAccessibleProjects",
-                signal: new AbortController().signal,
-                subjectId: SubjectId.fromUserId(userId),
-            },
-            async () => {
-                const allProjects: Project[] = [];
-                const teams = await this.organizationService.listOrganizationsByMember(userId, userId);
-                for (const team of teams) {
-                    allProjects.push(...(await this.projectsService.getProjects(userId, team.id)));
-                }
-                return allProjects;
-            },
-        );
     }
 
     private listenForWorkspaceInstanceUpdates(): void {
@@ -497,9 +476,9 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
 
     /**
      * Returns the descriptions of auth providers. This also controls the visibility of
-     * auth providers on the dashbard.
+     * auth providers on the dashboard.
      *
-     * If this call is unauthenticated (i.e. for anonumous users,) it returns only information
+     * If this call is unauthenticated (i.e. for anonymous users,) it returns only information
      * necessary for the Login page.
      *
      * If there are built-in auth providers configured, only these are returned.
@@ -1701,7 +1680,9 @@ export class GitpodServerImpl implements GitpodServerWithTracing, Disposable {
                     ctx,
                 );
 
-            this.disposables.pushAll([this.subscriber.listenForPrebuildUpdates(project.id, prebuildUpdateHandler)]);
+            this.disposables.pushAll([
+                this.subscriber.listenForProjectPrebuildUpdates(project.id, prebuildUpdateHandler),
+            ]);
         }
 
         return project;
diff --git a/components/server/src/workspace/workspace-starter.ts b/components/server/src/workspace/workspace-starter.ts
@@ -914,6 +914,7 @@ export class WorkspaceStarter {
                         prebuildID: prebuild.id,
                         projectID: prebuild.projectId,
                         workspaceID: workspace.id,
+                        organizationID: workspace.organizationId,
                     });
                 }
             }
diff --git a/components/ws-manager-bridge/src/prebuild-updater.ts b/components/ws-manager-bridge/src/prebuild-updater.ts
@@ -98,6 +98,7 @@ export class PrebuildUpdater {
                         prebuildID: updatedPrebuild.id,
                         status: updatedPrebuild.state,
                         workspaceID: workspaceId,
+                        organizationID: info.teamId,
                     });
                 }
             }
@@ -127,6 +128,7 @@ export class PrebuildUpdater {
                         prebuildID: prebuild.id,
                         status: prebuild.state,
                         workspaceID: instance.workspaceId,
+                        organizationID: info.teamId,
                     });
                 }
             }

Original file line number	Diff line number	Diff line change
`@@ -914,6 +914,7 @@ export class WorkspaceStarter {`
`914`	`914`	`prebuildID: prebuild.id,`
`915`	`915`	`projectID: prebuild.projectId,`
`916`	`916`	`workspaceID: workspace.id,`
	`917`	`+ organizationID: workspace.organizationId,`
`917`	`918`	`});`
`918`	`919`	`}`
`919`	`920`	`}`
Original file line number	Diff line number	Diff line change
`@@ -98,6 +98,7 @@ export class PrebuildUpdater {`
`98`	`98`	`prebuildID: updatedPrebuild.id,`
`99`	`99`	`status: updatedPrebuild.state,`
`100`	`100`	`workspaceID: workspaceId,`
	`101`	`+ organizationID: info.teamId,`
`101`	`102`	`});`
`102`	`103`	`}`
`103`	`104`	`}`
`@@ -127,6 +128,7 @@ export class PrebuildUpdater {`
`127`	`128`	`prebuildID: prebuild.id,`
`128`	`129`	`status: prebuild.state,`
`129`	`130`	`workspaceID: instance.workspaceId,`
	`131`	`+ organizationID: info.teamId,`
`130`	`132`	`});`
`131`	`133`	`}`
`132`	`134`	`}`