v3: self-hosting support for latest packages (#1319)

* display --profile flag after init if used

* add containerfile debug logs

* log all indexing errors in case of connection issues

* specify dockerfile syntax version

* add network flag for self-hosting

* move all self-hosting tags to latest

* add self-hosting update section

* manual setup subsection

* registry flag implies push

* add changeset

Author: nicktrn

.changeset/metal-geckos-pretend.md

@@ -0,0 +1,8 @@
+---
+"trigger.dev": patch
+---
+
+- Improve index error logging
+- Add network flag for self-hosted deploys
+- Fix checksum flag on some docker versions
+- Add Containerfile debug logs

docs/github-actions.mdx

@@ -181,7 +181,7 @@ jobs:
           TRIGGER_API_URL: ${{ secrets.TRIGGER_API_URL }}
         # deploy with additional flags
         run: |
-          npx trigger.dev@beta deploy --self-hosted --push
+          npx trigger.dev@latest deploy --self-hosted --push
 ```
 
 </CodeGroup>

docs/open-source-self-hosting.mdx

@@ -3,8 +3,6 @@ title: "Self-hosting"
 description: "You can self-host Trigger.dev on your own infrastructure."
 ---
 
-<Warning>Self-hosting does not support the latest CLI yet, you will have to continue using the `beta` tag for now.</Warning>
-
 ## Overview
 
 <Frame>
@@ -94,7 +92,7 @@ git checkout v3
 ./start.sh # hint: you can append -d to run in detached mode
 ```
 
-### Manual setup
+#### Manual
 
 Alternatively, you can follow these manual steps after cloning the docker repo:
 
@@ -176,7 +174,7 @@ docker login -u <your_dockerhub_username>
 5. You can now deploy v3 projects using the CLI with these flags:
 
 ```
-npx trigger.dev@beta deploy --self-hosted --push
+npx trigger.dev@latest deploy --self-hosted --push
 ```
 
 ## Part 2: Split services
@@ -286,12 +284,44 @@ echo "FORCE_CHECKPOINT_SIMULATION=0" >> .env
 ./stop.sh worker && ./start.sh worker
 ```
 
-## Telemetry
+## Updating
 
-By default, the Trigger.dev webapp sends telemetry data to our servers. This data is used to improve the product and is not shared with third parties. If you would like to opt-out of this, you can set the `TRIGGER_TELEMETRY_DISABLED` environment variable in your `.env` file. The value doesn't matter, it just can't be empty. For example:
+Once you have everything set up, you will periodically want to update your Docker images. You can easily do this by running the update script and restarting your services:
 
 ```bash
-TRIGGER_TELEMETRY_DISABLED=1
+./update.sh
+./stop.sh && ./start.sh
+```
+
+Sometimes, we will make more extensive changes that require pulling updated compose files, scripts, etc from our docker repo:
+
+```bash
+git pull
+./stop.sh && ./start.sh
+```
+
+Occasionally, you may also have to update your `.env` file, but we will try to keep these changes to a minimum. Check the `.env.example` file for new variables.
+
+### From beta
+
+If you're coming from the beta CLI package images, you will need to:
+- **Pull changes from our docker repo.** We've added a new container for [Electric](https://github.com/electric-sql/electric) and made some other improvements.
+
+```bash
+# wherever you cloned the docker repo
+git pull && ./stop.sh && ./start.sh
+```
+
+## Version locking
+
+There are several reasons to lock the version of your Docker images:
+- **Backwards compatibility.** We try our best to maintain compatibility with older CLI versions, but it's not always possible. If you don't want to update your CLI, you can lock your Docker images to that specific version.
+- **Ensuring full feature support.** Sometimes, new CLI releases will also require new or updated platform features. Running unlocked images can make any issues difficult to debug. Using a specific tag can help here as well.
+
+By default, the images will point at the latest versioned release via the `v3` tag. You can override this by specifying a different tag in your `.env` file. For example:
+
+```bash
+TRIGGER_IMAGE_TAG=v3.0.5
 ```
 
 ## CLI usage
@@ -303,7 +333,7 @@ This section highlights some of the CLI commands and options that are useful whe
 To avoid being redirected to the [Trigger.dev Cloud](https://cloud.trigger.dev) login page when using the CLI, you can specify the URL of your self-hosted instance with the `--api-url` or `-a` flag. For example:
 
 ```bash
-npx trigger.dev@beta login -a http://trigger.example.com
+npx trigger.dev@latest login -a http://trigger.example.com
 ```
 
 Once you've logged in, the CLI will remember your login details and you won't need to specify the URL again with other commands.
@@ -313,38 +343,38 @@ Once you've logged in, the CLI will remember your login details and you won't ne
 You can specify a custom profile when logging in. This allows you to easily use the CLI with our cloud product and your self-hosted instance at the same time. For example:
 
 ```
-npx trigger.dev@beta login -a http://trigger.example.com --profile my-profile
+npx trigger.dev@latest login -a http://trigger.example.com --profile my-profile
 ```
 
 You can then use this profile with other commands:
 
 ```
-npx trigger.dev@beta dev --profile my-profile
+npx trigger.dev@latest dev --profile my-profile
 ```
 
 To list all your profiles, use the `list-profiles` command:
 
 ```
-npx trigger.dev@beta list-profiles
+npx trigger.dev@latest list-profiles
 ```
 
 #### Verify login
 
 It can be useful to check you have successfully logged in to the correct instance. You can do this with the `whoami` command, which will also show the API URL:
 
 ```bash
-npx trigger.dev@beta whoami
+npx trigger.dev@latest whoami
 
 # with a custom profile
-npx trigger.dev@beta whoami --profile my-profile
+npx trigger.dev@latest whoami --profile my-profile
 ```
 
 ### Deploy
 
 On [Trigger.dev Cloud](https://cloud.trigger.dev), we build deployments remotely and push those images for you. When self-hosting you will have to do that locally yourself. This can be done with the `--self-hosted` and `--push` flags. For example:
 
 ```
-npx trigger.dev@beta deploy --self-hosted --push
+npx trigger.dev@latest deploy --self-hosted --push
 ```
 
 ### CI / GitHub Actions
@@ -353,3 +383,12 @@ When running the CLI in a CI environment, your login profiles won't be available
 variables to point at your self-hosted instance and authenticate.
 
 For more detailed instructions, see the [GitHub Actions guide](/github-actions).
+
+
+## Telemetry
+
+By default, the Trigger.dev webapp sends telemetry data to our servers. This data is used to improve the product and is not shared with third parties. If you would like to opt-out of this, you can set the `TRIGGER_TELEMETRY_DISABLED` environment variable in your `.env` file. The value doesn't matter, it just can't be empty. For example:
+
+```bash
+TRIGGER_TELEMETRY_DISABLED=1
+```

docs/snippets/cli-commands-deploy.mdx

@@ -81,33 +81,37 @@ These options are available on most commands.
 
 These options are typically used when [self-hosting](/open-source-self-hosting) or for local development.
 
-<ParamField body="Skip deploying the image" type="--skip-deploy | -D">
-  Load the built image into your local docker.
-</ParamField>
-
 <ParamField body="Self-hosted (builds locally)" type="--self-hosted">
   Builds and loads the image using your local docker. Use the `--registry` option to specify the
   registry to push the image to when using `--self-hosted`, or just use `--push` to push to the
   default registry.
 </ParamField>
 
+<ParamField body="Skip deploying the image" type="--skip-deploy | -D">
+  Load the built image into your local docker.
+</ParamField>
+
 <ParamField body="Load image" type="--load-image">
   Loads the image into your local docker after building it.
 </ParamField>
 
 <ParamField body="Registry" type="--registry">
-  Specify the registry to push the image to when using `--self-hosted`.
+  Specify the registry to push the image to when using `--self-hosted`. Will automatically enable `--push`.
 </ParamField>
 
 <ParamField body="Push image" type="--push">
-  When using the --self-hosted flag, push the image to the registry.
+  When using the `--self-hosted` flag, push the image to the registry.
 </ParamField>
 
 <ParamField body="Namespace" type="--namepsace">
   The namespace to use when pushing the image to the registry. For example, if pushing to Docker
   Hub, the namespace is your Docker Hub username.
 </ParamField>
 
+<ParamField body="Network" type="--network">
+  The networking mode for RUN instructions when using `--self-hosted`.
+</ParamField>
+
 ## Examples
 
 ### Push to Docker Hub (self-hosted)
@@ -118,7 +122,6 @@ An example of deploying to Docker Hub when using a self-hosted setup:
 npx trigger.dev@latest deploy \
   --self-hosted \
   --load-image \
-  --push \
   --registry docker.io \
   --namespace mydockerhubusername
 ```

docs/upgrading-beta.mdx

@@ -412,7 +412,6 @@ You can now specify a custom registry and namespace when deploying via a self-ho
 npx trigger.dev@latest deploy \
   --self-hosted \
   --load-image \
-  --push \
   --registry docker.io \
   --namespace mydockerhubusername
 ```

packages/cli-v3/src/build/buildWorker.ts

@@ -223,5 +223,10 @@ async function writeContainerfile(outputPath: string, buildManifest: BuildManife
     indexScript: buildManifest.indexControllerEntryPoint,
   });
 
-  await writeFile(join(outputPath, "Containerfile"), containerfile);
+  const containerfilePath = join(outputPath, "Containerfile");
+
+  logger.debug("Writing Containerfile", { containerfilePath });
+  logger.debug(containerfile);
+
+  await writeFile(containerfilePath, containerfile);
 }

packages/cli-v3/src/commands/deploy.ts

@@ -55,6 +55,7 @@ const DeployCommandOptions = CommonCommandOptions.extend({
   skipUpdateCheck: z.boolean().default(false),
   noCache: z.boolean().default(false),
   envFile: z.string().optional(),
+  network: z.enum(["default", "none", "host"]).optional(),
 });
 
 type DeployCommandOptions = z.infer<typeof DeployCommandOptions>;
@@ -144,6 +145,7 @@ export function configureDeployCommand(program: Command) {
         "If provided, will save logs even for successful builds"
       ).hideHelp()
     )
+    .option("--network <mode>", "The networking mode for RUN instructions when using --self-hosted")
     .action(async (path, options) => {
       await handleTelemetry(async () => {
         await printStandloneInitialBanner(true);
@@ -349,6 +351,7 @@ async function _deployCommand(dir: string, options: DeployCommandOptions) {
     authAccessToken: authorization.auth.accessToken,
     compilationPath: destination.path,
     buildEnvVars: buildManifest.build.env,
+    network: options.network,
   });
 
   logger.debug("Build result", buildResult);

packages/cli-v3/src/commands/init.ts

@@ -194,9 +194,7 @@ async function _initCommand(dir: string, options: InitCommandOptions) {
   log.info("Next steps:");
   log.info(
     `   1. To start developing, run ${chalk.green(
-      `npx trigger.dev@${options.tag} dev${
-        options.apiUrl === CLOUD_API_URL ? "" : ` -a ${options.apiUrl}`
-      }`
+      `npx trigger.dev@${options.tag} dev${options.profile ? "" : ` --profile ${options.profile}`}`
     )} in your project directory`
   );
   log.info(`   2. Visit your ${projectDashboard} to view your newly created tasks.`);

packages/cli-v3/src/deploy/buildImage.ts

@@ -14,6 +14,7 @@ export interface BuildImageOptions {
   // Self-hosted specific options
   push: boolean;
   registry?: string;
+  network?: string;
 
   // Non-self-hosted specific options
   loadImage?: boolean;
@@ -84,6 +85,7 @@ export async function buildImage(options: BuildImageOptions) {
       apiUrl,
       apiKey,
       buildEnvVars,
+      network: options.network,
     });
   }
 
@@ -277,6 +279,7 @@ interface SelfHostedBuildImageOptions {
   noCache?: boolean;
   extraCACerts?: string;
   buildEnvVars?: Record<string, string | undefined>;
+  network?: string;
 }
 
 async function selfHostedBuildImage(
@@ -295,6 +298,7 @@ async function selfHostedBuildImage(
     options.noCache ? "--no-cache" : undefined,
     "--platform",
     options.buildPlatform,
+    ...(options.network ? ["--network", options.network] : []),
     "--build-arg",
     `TRIGGER_PROJECT_ID=${options.projectId}`,
     "--build-arg",
@@ -458,7 +462,7 @@ async function generateBunContainerfile(options: GenerateContainerfileOptions) {
     " "
   );
 
-  return `
+  return `# syntax=docker/dockerfile:1
 FROM imbios/bun-node:22-debian AS base
 
 ${baseInstructions}
@@ -563,7 +567,7 @@ async function generateNodeContainerfile(options: GenerateContainerfileOptions)
     " "
   );
 
-  return `
+  return `# syntax=docker/dockerfile:1
 FROM node:21-bookworm-slim@sha256:99afef5df7400a8d118e0504576d32ca700de5034c4f9271d2ff7c91cc12d170 AS base
 
 ${baseInstructions}

packages/cli-v3/src/entryPoints/deploy-index-controller.ts

@@ -104,6 +104,8 @@ async function indexDeployment({
   } catch (error) {
     const serialiedIndexError = serializeIndexingError(error, stderr.join("\n"));
 
+    console.error("Failed to index deployment", serialiedIndexError);
+
     await cliApiClient.failDeployment(deploymentId, { error: serialiedIndexError });
 
     process.exit(1);