fix supervisor api domain for split setups

nicktrn · nicktrn · commit 22a6035ced77 · 2025-03-07T11:39:30.000Z
diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts
@@ -4,18 +4,31 @@ import { z } from "zod";
 import { getDockerHostDomain } from "./util.js";
 
 const Env = z.object({
-  // This will come from `status.hostIP` in k8s
-  WORKER_HOST_IP: z.string().default(getDockerHostDomain()),
-  TRIGGER_API_URL: z.string().url(),
-  TRIGGER_WORKER_TOKEN: z.string(),
   // This will come from `spec.nodeName` in k8s
   TRIGGER_WORKER_INSTANCE_NAME: z.string().default(randomUUID()),
+
+  // Required settings
+  TRIGGER_API_URL: z.string().url(),
+  TRIGGER_WORKER_TOKEN: z.string(),
   MANAGED_WORKER_SECRET: z.string(),
-  TRIGGER_WORKLOAD_API_PORT: z.coerce.number().default(8020),
-  TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020),
+
+  // Workload API settings (coordinator mode) - the workload API is what the run controller connects to
+  TRIGGER_WORKLOAD_API_ENABLED: z.coerce.boolean().default(true),
+  TRIGGER_WORKLOAD_API_PROTOCOL: z
+    .string()
+    .transform((s) => z.enum(["http", "https"]).parse(s.toLowerCase()))
+    .default("http"),
+  TRIGGER_WORKLOAD_API_DOMAIN: z.string().default(getDockerHostDomain()), // If unset, will use orchestrator-specific default
+  TRIGGER_WORKLOAD_API_PORT_INTERNAL: z.coerce.number().default(8020), // This is the port the workload API listens on
+  TRIGGER_WORKLOAD_API_PORT_EXTERNAL: z.coerce.number().default(8020), // This is the exposed port passed to the run controller
+
+  // Dequeue settings (provider mode)
+  TRIGGER_DEQUEUE_ENABLED: z.coerce.boolean().default(true),
+  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(1000),
+
+  // Optional services
   TRIGGER_WARM_START_URL: z.string().optional(),
   TRIGGER_CHECKPOINT_URL: z.string().optional(),
-  TRIGGER_DEQUEUE_INTERVAL_MS: z.coerce.number().int().default(1000),
 
   // Used by the workload manager, e.g docker/k8s
   DOCKER_NETWORK: z.string().default("host"),
diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts
@@ -29,7 +29,9 @@ class ManagedSupervisor {
   private readonly warmStartUrl = env.TRIGGER_WARM_START_URL;
 
   constructor() {
-    const workerApiUrl = `http://${env.WORKER_HOST_IP}:${env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL}`;
+    const workloadApiProtocol = env.TRIGGER_WORKLOAD_API_PROTOCOL;
+    const workloadApiDomain = env.TRIGGER_WORKLOAD_API_DOMAIN;
+    const workloadApiPortExternal = env.TRIGGER_WORKLOAD_API_PORT_EXTERNAL;
 
     if (this.warmStartUrl) {
       this.logger.log("[ManagedWorker] 🔥 Warm starts enabled", {
@@ -40,13 +42,17 @@ class ManagedSupervisor {
     if (this.isKubernetes) {
       this.resourceMonitor = new KubernetesResourceMonitor(createK8sApi(), "");
       this.workloadManager = new KubernetesWorkloadManager({
-        workerApiUrl,
+        workloadApiProtocol,
+        workloadApiDomain,
+        workloadApiPort: workloadApiPortExternal,
         warmStartUrl: this.warmStartUrl,
       });
     } else {
       this.resourceMonitor = new DockerResourceMonitor(new Docker());
       this.workloadManager = new DockerWorkloadManager({
-        workerApiUrl,
+        workloadApiProtocol,
+        workloadApiDomain,
+        workloadApiPort: workloadApiPortExternal,
         warmStartUrl: this.warmStartUrl,
       });
     }
@@ -57,6 +63,8 @@ class ManagedSupervisor {
       instanceName: env.TRIGGER_WORKER_INSTANCE_NAME,
       managedWorkerSecret: env.MANAGED_WORKER_SECRET,
       dequeueIntervalMs: env.TRIGGER_DEQUEUE_INTERVAL_MS,
+      queueConsumerEnabled: env.TRIGGER_DEQUEUE_ENABLED,
+      runNotificationsEnabled: env.TRIGGER_WORKLOAD_API_ENABLED,
       preDequeue: async () => {
         if (this.isKubernetes) {
           // TODO: Test k8s resource monitor and remove this
@@ -180,7 +188,7 @@ class ManagedSupervisor {
 
     // Responds to workload requests only
     this.workloadServer = new WorkloadServer({
-      port: env.TRIGGER_WORKLOAD_API_PORT,
+      port: env.TRIGGER_WORKLOAD_API_PORT_INTERNAL,
       workerClient: this.workerSession.httpClient,
       checkpointClient: this.checkpointClient,
     });
diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts
@@ -6,12 +6,18 @@ import {
 } from "./types.js";
 import { x } from "tinyexec";
 import { env } from "../env.js";
-import { RunnerId } from "../util.js";
+import { getDockerHostDomain, RunnerId } from "../util.js";
 
 export class DockerWorkloadManager implements WorkloadManager {
   private readonly logger = new SimpleStructuredLogger("docker-workload-provider");
 
-  constructor(private opts: WorkloadManagerOptions) {}
+  constructor(private opts: WorkloadManagerOptions) {
+    if (opts.workloadApiDomain) {
+      this.logger.warn("[DockerWorkloadProvider] ⚠️ Custom workload API domain", {
+        domain: opts.workloadApiDomain,
+      });
+    }
+  }
 
   async create(opts: WorkloadManagerCreateOptions) {
     this.logger.log("[DockerWorkloadProvider] Creating container", { opts });
@@ -24,7 +30,9 @@ export class DockerWorkloadManager implements WorkloadManager {
       `--env=TRIGGER_ENV_ID=${opts.envId}`,
       `--env=TRIGGER_RUN_ID=${opts.runFriendlyId}`,
       `--env=TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`,
-      `--env=TRIGGER_WORKER_API_URL=${this.opts.workerApiUrl}`,
+      `--env=TRIGGER_SUPERVISOR_API_PROTOCOL=${this.opts.workloadApiProtocol}`,
+      `--env=TRIGGER_SUPERVISOR_API_PORT=${this.opts.workloadApiPort}`,
+      `--env=TRIGGER_SUPERVISOR_API_DOMAIN=${this.opts.workloadApiDomain ?? getDockerHostDomain()}`,
       `--env=TRIGGER_WORKER_INSTANCE_NAME=${env.TRIGGER_WORKER_INSTANCE_NAME}`,
       `--env=OTEL_EXPORTER_OTLP_ENDPOINT=${env.OTEL_EXPORTER_OTLP_ENDPOINT}`,
       `--env=TRIGGER_RUNNER_ID=${runnerId}`,
diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts
@@ -23,6 +23,12 @@ export class KubernetesWorkloadManager implements WorkloadManager {
 
   constructor(private opts: WorkloadManagerOptions) {
     this.k8s = createK8sApi();
+
+    if (opts.workloadApiDomain) {
+      this.logger.warn("[KubernetesWorkloadManager] ⚠️ Custom workload API domain", {
+        domain: opts.workloadApiDomain,
+      });
+    }
   }
 
   async create(opts: WorkloadManagerCreateOptions) {
@@ -72,8 +78,26 @@ export class KubernetesWorkloadManager implements WorkloadManager {
                     value: opts.snapshotFriendlyId,
                   },
                   {
-                    name: "TRIGGER_WORKER_API_URL",
-                    value: this.opts.workerApiUrl,
+                    name: "TRIGGER_SUPERVISOR_API_PROTOCOL",
+                    value: this.opts.workloadApiProtocol,
+                  },
+                  {
+                    name: "TRIGGER_SUPERVISOR_API_PORT",
+                    value: `${this.opts.workloadApiPort}`,
+                  },
+                  {
+                    name: "TRIGGER_SUPERVISOR_API_DOMAIN",
+                    ...(this.opts.workloadApiDomain
+                      ? {
+                          value: this.opts.workloadApiDomain,
+                        }
+                      : {
+                          valueFrom: {
+                            fieldRef: {
+                              fieldPath: "status.hostIP",
+                            },
+                          },
+                        }),
                   },
                   {
                     name: "TRIGGER_WORKER_INSTANCE_NAME",
diff --git a/apps/supervisor/src/workloadManager/types.ts b/apps/supervisor/src/workloadManager/types.ts
@@ -1,7 +1,9 @@
 import { type EnvironmentType, type MachinePreset } from "@trigger.dev/core/v3";
 
 export interface WorkloadManagerOptions {
-  workerApiUrl: string;
+  workloadApiProtocol: "http" | "https";
+  workloadApiDomain?: string; // If unset, will use orchestrator-specific default
+  workloadApiPort: number;
   warmStartUrl?: string;
 }
 
diff --git a/packages/cli-v3/src/entryPoints/managed-run-controller.ts b/packages/cli-v3/src/entryPoints/managed-run-controller.ts
@@ -36,7 +36,9 @@ const Env = z.object({
   NODE_EXTRA_CA_CERTS: z.string().optional(),
 
   // Set at runtime
-  TRIGGER_WORKER_API_URL: z.string().url(),
+  TRIGGER_SUPERVISOR_API_PROTOCOL: z.enum(["http", "https"]),
+  TRIGGER_SUPERVISOR_API_DOMAIN: z.string(),
+  TRIGGER_SUPERVISOR_API_PORT: z.coerce.number(),
   TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`),
   TRIGGER_ENV_ID: z.string(),
   TRIGGER_RUN_ID: z.string().optional(), // This is only useful for cold starts
@@ -84,6 +86,8 @@ class ManagedRunController {
   private readonly snapshotPoller: HeartbeatService;
   private readonly snapshotPollIntervalSeconds: number;
 
+  private readonly workerApiUrl: string;
+
   private state:
     | {
         phase: "RUN";
@@ -246,8 +250,10 @@ class ManagedRunController {
     this.heartbeatIntervalSeconds = opts.heartbeatIntervalSeconds || 30;
     this.snapshotPollIntervalSeconds = 5;
 
+    this.workerApiUrl = `${env.TRIGGER_SUPERVISOR_API_PROTOCOL}://${env.TRIGGER_SUPERVISOR_API_DOMAIN}:${env.TRIGGER_SUPERVISOR_API_PORT}`;
+
     this.httpClient = new WorkloadHttpClient({
-      workerApiUrl: env.TRIGGER_WORKER_API_URL,
+      workerApiUrl: this.workerApiUrl,
       deploymentId: env.TRIGGER_DEPLOYMENT_ID,
       runnerId: env.TRIGGER_RUNNER_ID,
     });
@@ -746,8 +752,7 @@ class ManagedRunController {
   }
 
   createSocket() {
-    const wsUrl = new URL(env.TRIGGER_WORKER_API_URL);
-    wsUrl.pathname = "/workload";
+    const wsUrl = new URL("/workload", this.workerApiUrl);
 
     this.socket = io(wsUrl.href, {
       transports: ["websocket"],

Original file line number	Diff line number	Diff line change
`@@ -1,7 +1,9 @@`
`1`	`1`	`import { type EnvironmentType, type MachinePreset } from "@trigger.dev/core/v3";`
`2`	`2`
`3`	`3`	`export interface WorkloadManagerOptions {`
`4`		`- workerApiUrl: string;`
	`4`	`+ workloadApiProtocol: "http" \| "https";`
	`5`	`+ workloadApiDomain?: string; // If unset, will use orchestrator-specific default`
	`6`	`+ workloadApiPort: number;`
`5`	`7`	`warmStartUrl?: string;`
`6`	`8`	`}`
`7`	`9`