Add gRPC retries

but not working with Fabric because ALB sets metadata
which gets interpreted as 'pushback' by grpc-js.

Author: lionello

client.ts

@@ -0,0 +1,43 @@
+import * as grpc from "@grpc/grpc-js";
+import * as fabric from "./protos/io/defang/v1/fabric_grpc_pb";
+import { Status } from "@grpc/grpc-js/build/src/constants";
+
+function hasPort(url: string): boolean {
+  return /:\d+$/.test(url);
+}
+
+export function newClient(
+  fabricDns: string,
+  accessToken: string
+): fabric.FabricControllerClient {
+  const serviceConfig: grpc.ServiceConfig = {
+    loadBalancingConfig: [],
+    methodConfig: [
+      {
+        name: [{ service: "io.defang.v1.FabricController" }],
+        retryPolicy: {
+          maxAttempts: 5,
+          initialBackoff: "1s",
+          maxBackoff: "10s",
+          backoffMultiplier: 2,
+          retryableStatusCodes: [Status.UNAVAILABLE, Status.INTERNAL],
+        },
+      },
+    ],
+  };
+  const noTenant = fabricDns.replace(/^.*@/, "");
+  const withPort = hasPort(noTenant) ? noTenant : `${noTenant}:443`;
+  return new fabric.FabricControllerClient(
+    withPort,
+    grpc.credentials.combineChannelCredentials(
+      grpc.credentials.createSsl(),
+      grpc.credentials.createFromMetadataGenerator((_, callback) => {
+        const metadata = new grpc.Metadata();
+        // TODO: automatically generate a new token once it expires
+        metadata.set("authorization", "Bearer " + accessToken);
+        callback(null, metadata);
+      })
+    ),
+    { "grpc.service_config": JSON.stringify(serviceConfig) }
+  );
+}

index.ts

@@ -7,6 +7,7 @@ import { dirname, join } from "path";
 import { promises as stream } from "stream";
 import assert = require("assert");
 
+import { newClient } from "./client";
 import * as fabric from "./protos/io/defang/v1/fabric_grpc_pb";
 import * as pb from "./protos/io/defang/v1/fabric_pb";
 import { createTarball, uploadTarball } from "./upload";
@@ -74,27 +75,11 @@ export function setAccessToken(token: string) {
   accessToken = token;
 }
 
-function hasPort(url: string): boolean {
-  return /:\d+$/.test(url);
-}
-
 // Connect to our gRPC server
 async function connect(
   fabricDns: string
 ): Promise<fabric.FabricControllerClient> {
-  const withoutTenant = fabricDns.replace(/^.*@/, "");
-  const client = new fabric.FabricControllerClient(
-    hasPort(withoutTenant) ? withoutTenant : `${withoutTenant}:443`,
-    grpc.credentials.combineChannelCredentials(
-      grpc.credentials.createSsl(),
-      grpc.credentials.createFromMetadataGenerator((_, callback) => {
-        const metadata = new grpc.Metadata();
-        // TODO: automatically generate a new token once it expires
-        metadata.set("authorization", "Bearer " + getAccessToken(fabricDns));
-        callback(null, metadata);
-      })
-    )
-  );
+  const client = newClient(fabricDns, getAccessToken(fabricDns));
   await new Promise<void>((resolve, reject) =>
     client.waitForReady(Date.now() + 5000, (err) =>
       err ? reject(err) : resolve()
@@ -301,7 +286,7 @@ async function upsert(
         if (secret.value === undefined) {
           return;
         }
-        const sv = new pb.SecretValue();
+        const sv = new pb.PutConfigRequest();
         sv.setProject(inputs.project);
         sv.setName(secret.source);
         sv.setValue(secret.value);
@@ -868,7 +853,7 @@ export class DefangService extends pulumi.dynamic.Resource {
       args.fabricDNS = defaultFabric;
     }
     if (!args.project) {
-      args.project = pulumi.getProject()+"-"+pulumi.getStack();
+      args.project = pulumi.getProject() + "-" + pulumi.getStack();
     }
     super(
       defangServiceProvider,

package-lock.json

@@ -7,8 +7,8 @@
   },
   "dependencies": {
     "@balena/dockerignore": "^1.0.2",
-    "@grpc/grpc-js": "^1.8.17",
-    "google-protobuf": "^3.21.2",
+    "@grpc/grpc-js": "^1.12.2",
+    "google-protobuf": "^3.5.0",
     "node-fetch": "^3.3.1",
     "tar": "^6.1.15"
   },

package.json

@@ -0,0 +1,13 @@
+import "mocha";
+import { newClient } from "../client";
+import * as pb from "../protos/io/defang/v1/fabric_pb";
+
+describe("#newClient", () => {
+  it("connect to fabric-prod1.defang.dev", function (done) {
+    const client = newClient("fabric-prod1.defang.dev", "");
+
+    const trackRequest = new pb.TrackRequest();
+    trackRequest.setEvent("Pulumi-Defang Test");
+    client.track(trackRequest, (err) => (err ? done(err) : done()));
+  });
+});

tests/client.test.ts

@@ -11,6 +11,8 @@ const fromDir = new DefangService("test1", {
   healthcheck: {
     test: ["CMD", "wget", "--spider", "-q", "http://localhost/health"],
     retries: 1,
+    timeout: 3,
+    interval: 5,
   },
   waitForSteadyState: true,
 });

tests/index.ts

@@ -11,11 +11,11 @@
         "typescript": "^5.1.3"
     },
     "dependencies": {
-        "@defang-io/pulumi-defang": "^0.16.1",
+        "@defang-io/pulumi-defang": "^0.17.2",
         "@pulumi/pulumi": "~3.117.0"
     },
     "scripts": {
         "test": "mocha --require ts-node/register *.test.ts",
-        "posttest": "pulumi login file://. && PULUMI_CONFIG_PASSPHRASE= pulumi up -s test --yes ; pulumi destroy -s test --yes"
+        "posttest": "pulumi login file://. && PULUMI_CONFIG_PASSPHRASE= pulumi up -s test --yes --refresh ; pulumi destroy -s test --yes"
     }
 }

tests/package-lock.json

@@ -5,7 +5,7 @@ import { dirname, resolve } from "path";
 import * as upload from "../upload";
 import * as tar from "tar";
 
-describe("createTarball", () => {
+describe("#createTarball", () => {
   const expected = [
     "./",
     "./Dockerfile.test",