[public-api] improve error handling (#18739)

* [dashboard] add exponential backoff for streaming emulation

* [dashboard] add source map resolver

* [server] fix actual bug

* [public-api] compile to es6

to prevent duplicate @bufbuild modules in dashboard

* [public-api] export ESM modules

Author: akosyakov

components/dashboard/resolver.js

@@ -0,0 +1,90 @@
+/**
+ * Copyright (c) 2023 Gitpod GmbH. All rights reserved.
+ * Licensed under the GNU Affero General Public License (AGPL).
+ * See License.AGPL.txt in the project root for license information.
+ */
+//@ts-check
+const path = require('path');
+const fetch = require('node-fetch').default;
+const { SourceMapConsumer } = require('source-map');
+
+const sourceMapCache = {};
+
+function extractJsUrlFromLine(line) {
+    const match = line.match(/https?:\/\/[^\s]+\.js/);
+    return match ? match[0] : null;
+}
+
+async function fetchSourceMap(jsUrl) {
+    // Use cached source map if available
+    if (sourceMapCache[jsUrl]) {
+        return sourceMapCache[jsUrl];
+    }
+
+    const jsResponse = await fetch(jsUrl);
+    const jsContent = await jsResponse.text();
+
+    // Extract source map URL from the JS file
+    const mapUrlMatch = jsContent.match(/\/\/#\s*sourceMappingURL=(.+)/);
+    if (!mapUrlMatch) {
+        throw new Error('Source map URL not found');
+    }
+
+    const mapUrl = new URL(mapUrlMatch[1], jsUrl).href;  // Resolve relative URL
+    const mapResponse = await fetch(mapUrl);
+    const mapData = await mapResponse.json();
+
+    // Cache the fetched source map
+    sourceMapCache[jsUrl] = mapData;
+
+    return mapData;
+}
+
+const BASE_PATH = '/workspace/gitpod/components';
+
+async function resolveLine(line) {
+    const jsUrl = extractJsUrlFromLine(line);
+    if (!jsUrl) return line;
+
+    const rawSourceMap = await fetchSourceMap(jsUrl);
+    const matches = line.match(/at (?:([\S]+) )?\(?(https?:\/\/[^\s]+\.js):(\d+):(\d+)\)?/);
+
+    if (!matches) {
+        return line;
+    }
+
+    const functionName = matches[1] || '';
+    const lineNum = Number(matches[3]);
+    const colNum = Number(matches[4]);
+
+    const consumer = new SourceMapConsumer(rawSourceMap);
+    const originalPosition = consumer.originalPositionFor({ line: lineNum, column: colNum });
+
+    if (originalPosition && originalPosition.source) {
+        const fullPath = path.join(BASE_PATH, originalPosition.source);
+        const originalFunctionName = originalPosition.name || functionName;
+        return `    at ${originalFunctionName} (${fullPath}:${originalPosition.line}:${originalPosition.column})`;
+    }
+
+    return line;
+}
+
+
+let obfuscatedTrace = '';
+
+process.stdin.on('data', function(data) {
+    obfuscatedTrace += data;
+});
+
+process.stdin.on('end', async function() {
+    const lines = obfuscatedTrace.split('\n');
+    const resolvedLines = await Promise.all(lines.map(resolveLine));
+    const resolvedTrace = resolvedLines.join('\n');
+    console.log('\nResolved Stack Trace:\n');
+    console.log(resolvedTrace);
+});
+
+if (process.stdin.isTTY) {
+    console.error("Please provide the obfuscated stack trace either as a multi-line input or from a file.");
+    process.exit(1);
+}

components/dashboard/src/service/service.tsx

@@ -110,6 +110,10 @@ function testPublicAPI(service: any): void {
     });
     (async () => {
         const grpcType = "server-stream";
+        const MAX_BACKOFF = 60000;
+        const BASE_BACKOFF = 3000;
+        let backoff = BASE_BACKOFF;
+
         // emulates server side streaming with public API
         while (true) {
             const isTest = await getExperimentsClient().getValueAsync("public_api_dummy_reliability_test", false, {
@@ -121,15 +125,21 @@ function testPublicAPI(service: any): void {
                     let previousCount = 0;
                     for await (const reply of helloService.lotsOfReplies({ previousCount })) {
                         previousCount = reply.count;
+                        backoff = BASE_BACKOFF;
                     }
                 } catch (e) {
                     console.error(e, {
                         userId: user?.id,
                         grpcType,
                     });
+                    backoff = Math.min(2 * backoff, MAX_BACKOFF);
                 }
+            } else {
+                backoff = BASE_BACKOFF;
             }
-            await new Promise((resolve) => setTimeout(resolve, 3000));
+            const jitter = Math.random() * 0.3 * backoff;
+            const delay = backoff + jitter;
+            await new Promise((resolve) => setTimeout(resolve, delay));
         }
     })();
 }

components/public-api/typescript/package.json

@@ -4,11 +4,31 @@
   "license": "AGPL-3.0",
   "main": "./lib/index.js",
   "types": "./lib/index.d.ts",
+  "module": "./lib/esm/index.js",
   "files": [
     "lib"
   ],
+  "exports": {
+    ".": {
+      "types": "./lib/index.d.ts",
+      "import": "./lib/esm/index.js",
+      "require": "./lib/index.js"
+    },
+    "./lib/*": {
+      "types": "./lib/*.d.ts",
+      "import": "./lib/esm/*.js",
+      "require": "./lib/*.js"
+    },
+    "./lib/gitpod/experimental/v1": {
+      "types": "./lib/gitpod/experimental/v1/index.d.ts",
+      "import": "./lib/esm/gitpod/experimental/v1/index.js",
+      "require": "./lib/gitpod/experimental/v1/index.js"
+    }
+  },
   "scripts": {
-    "build": "mkdir -p lib; tsc",
+    "build": "yarn run build:cjs && yarn run build:esm",
+    "build:cjs": "tsc",
+    "build:esm": "tsc --module es2015 --outDir ./lib/esm",
     "watch": "leeway exec --package .:lib --transitive-dependencies --filter-type yarn --components --parallel -- tsc -w --preserveWatchOutput",
     "test": "mocha './**/*.spec.ts' --exclude './node_modules/**' --exit",
     "test:brk": "yarn test --inspect-brk"
@@ -27,11 +47,11 @@
   "dependencies": {
     "@bufbuild/connect": "^0.13.0",
     "@bufbuild/protobuf": "^0.1.1",
-    "@bufbuild/protoc-gen-connect-web": "^0.2.1",
-    "@bufbuild/protoc-gen-es": "^0.1.1",
     "prom-client": "^14.2.0"
   },
   "devDependencies": {
+    "@bufbuild/protoc-gen-connect-web": "^0.2.1",
+    "@bufbuild/protoc-gen-es": "^0.1.1",
     "@testdeck/mocha": "0.1.2",
     "@types/chai": "^4.1.2",
     "@types/node": "^16.11.0",

components/public-api/typescript/tsconfig.json

@@ -3,6 +3,7 @@
         "rootDir": "src",
         "experimentalDecorators": true,
         "outDir": "lib",
+        "declarationDir": "lib",
         "lib": [
             "es6",
             "esnext.asynciterable",
@@ -14,7 +15,7 @@
         "emitDecoratorMetadata": true,
         "strictPropertyInitialization": false,
         "downlevelIteration": true,
-        "module": "commonjs",
+        "module": "CommonJS",
         "moduleResolution": "node",
         "target": "es6",
         "jsx": "react",

components/server/src/api/server.ts

@@ -24,7 +24,6 @@ import { APIStatsService } from "./stats";
 import { APITeamsService } from "./teams";
 import { APIUserService } from "./user";
 import { APIWorkspacesService } from "./workspaces";
-import { Deferred } from "@gitpod/gitpod-protocol/lib/util/deferred";
 
 function service<T extends ServiceType>(type: T, impl: ServiceImpl<T>): [T, ServiceImpl<T>] {
     return [type, impl];
@@ -55,7 +54,7 @@ export class API {
         this.register(app);
 
         const server = app.listen(3001, () => {
-            log.info(`Connect Public API server listening on port: ${(server.address() as AddressInfo).port}`);
+            log.info(`public api: listening on port: ${(server.address() as AddressInfo).port}`);
         });
 
         return server;
@@ -126,13 +125,22 @@ export class API {
 
                     grpcServerStarted.labels(grpc_service, grpc_method, grpc_type).inc();
                     const stopTimer = grpcServerHandling.startTimer({ grpc_service, grpc_method, grpc_type });
-                    const deferred = new Deferred<ConnectError | undefined>();
-                    // eslint-disable-next-line @typescript-eslint/no-floating-promises
-                    deferred.promise.then((err) => {
+                    const done = (err?: ConnectError) => {
                         const grpc_code = err ? Code[err.code] : "OK";
                         grpcServerHandled.labels(grpc_service, grpc_method, grpc_type, grpc_code).inc();
                         stopTimer({ grpc_code });
-                    });
+                    };
+                    const handleError = (reason: unknown) => {
+                        let err = ConnectError.from(reason, Code.Internal);
+                        if (reason != err && err.code === Code.Internal) {
+                            console.error("public api: unexpected internal error", reason);
+                            // don't leak internal errors to a user
+                            // TODO(ak) instead surface request id
+                            err = ConnectError.from(`please check server logs`, Code.Internal);
+                        }
+                        done(err);
+                        throw err;
+                    };
 
                     const context = args[1] as HandlerContext;
                     async function call<T>(): Promise<T> {
@@ -146,12 +154,10 @@ export class API {
                             try {
                                 const promise = await call<Promise<any>>();
                                 const result = await promise;
-                                deferred.resolve(undefined);
+                                done();
                                 return result;
                             } catch (e) {
-                                const err = ConnectError.from(e);
-                                deferred.resolve(e);
-                                throw err;
+                                handleError(e);
                             }
                         })();
                     }
@@ -161,11 +167,9 @@ export class API {
                             for await (const item of generator) {
                                 yield item;
                             }
-                            deferred.resolve(undefined);
+                            done();
                         } catch (e) {
-                            const err = ConnectError.from(e);
-                            deferred.resolve(err);
-                            throw err;
+                            handleError(e);
                         }
                     })();
                 };
@@ -174,7 +178,7 @@ export class API {
     }
 
     private async verify(context: HandlerContext) {
-        const user = await this.sessionHandler.verify(context.requestHeader.get("cookie"));
+        const user = await this.sessionHandler.verify(context.requestHeader.get("cookie") || "");
         if (!user) {
             throw new ConnectError("unauthenticated", Code.Unauthenticated);
         }