diff --git a/docs/config/config-file.mdx b/docs/config/config-file.mdx index 7acef7f0f6..2f013a03df 100644 --- a/docs/config/config-file.mdx +++ b/docs/config/config-file.mdx @@ -315,479 +315,36 @@ Build extension allow you to hook into the build system and customize the build #### additionalFiles -Import the `additionalFiles` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { additionalFiles } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }), - ], - }, -}); -``` - -This will copy the files specified in the `files` array to the build directory. The `files` array can contain globs. The output paths will match the path of the file, relative to the root of the project. - -The root of the project is the directory that contains the trigger.config.ts file +See the [additionalFiles documentation](/config/extensions/additionalFiles) for more information. #### `additionalPackages` -Import the `additionalPackages` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { additionalPackages } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [additionalPackages({ packages: ["wrangler"] })], - }, -}); -``` - -This allows you to include additional packages in the build that are not automatically included via imports. This is useful if you want to install a package that includes a CLI tool that you want to invoke in your tasks via `exec`. We will try to automatically resolve the version of the package but you can specify the version by using the `@` symbol: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [additionalPackages({ packages: ["wrangler@1.19.0"] })], - }, -}); -``` +See the [additionalPackages documentation](/config/extensions/additionalPackages) for more information. #### `emitDecoratorMetadata` -If you need support for the `emitDecoratorMetadata` typescript compiler option, import the `emitDecoratorMetadata` build extension and use it in your `trigger.config.ts` file: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [emitDecoratorMetadata()], - }, -}); -``` - -This is usually required if you are using certain ORMs, like TypeORM, that require this option to be enabled. It's not enabled by default because there is a performance cost to enabling it. - - - emitDecoratorMetadata works by hooking into the esbuild bundle process and using the TypeScript - compiler API to compile files where we detect the use of decorators. This means you must have - `emitDecoratorMetadata` enabled in your `tsconfig.json` file, as well as `typescript` installed in - your `devDependencies`. - +See the [emitDecoratorMetadata documentation](/config/extensions/emitDecoratorMetadata) for more information. #### Prisma -If you are using Prisma, you should use the prisma build extension. - -- Automatically handles copying Prisma files to the build directory -- Generates the Prisma client during the deploy process -- Optionally will migrate the database during the deploy process -- Support for TypedSQL and multiple schema files -- You can use `prismaSchemaFolder` to specify just the directory containing your schema file, instead of the full path -- You can add the extension twice if you have multiple separate schemas in the same project (example below) - -You can use it for a simple Prisma setup like this: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - prismaExtension({ - version: "5.19.0", // optional, we'll automatically detect the version if not provided - schema: "prisma/schema.prisma", - }), - ], - }, -}); -``` - - - This does not have any effect when running the `dev` command, only when running the `deploy` - command. - - -If you want to also run migrations during the build process, you can pass in the `migrate` option: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - migrate: true, - directUrlEnvVarName: "DATABASE_URL_UNPOOLED", // optional - the name of the environment variable that contains the direct database URL if you are using a direct database URL - }), - ], - }, -}); -``` - -If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated. Some examples where you may need to do this include when using the `prisma-kysely` or `prisma-json-types-generator` generators. - - - -```prisma schema.prisma -datasource db { - provider = "postgresql" - url = env("DATABASE_URL") - directUrl = env("DATABASE_URL_UNPOOLED") -} - -// We only want to generate the prisma-client-js generator -generator client { - provider = "prisma-client-js" -} - -generator kysely { - provider = "prisma-kysely" - output = "../../src/kysely" - enumFileName = "enums.ts" - fileName = "types.ts" -} - -generator json { - provider = "prisma-json-types-generator" -} -``` - -```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - clientGenerator: "client", - }), - ], - }, -}); -``` - - - -If you are using [TypedSQL](https://www.prisma.io/typedsql), you'll need to enable it via the `typedSql` option: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - prismaExtension({ - schema: "prisma/schema.prisma", - typedSql: true, - }), - ], - }, -}); -``` - - - The `prismaExtension` will inject the `DATABASE_URL` environment variable into the build process. Learn more about setting environment variables for deploying in our [Environment Variables](/deploy-environment-variables) guide. - -These environment variables are only used during the build process and are not embedded in the final container image. - - - -If you have multiple separate schemas in the same project you can add the extension multiple times: - -```ts -prismaExtension({ - schema: 'prisma/schema/main.prisma', - version: '6.2.0', - migrate: false, -}), -prismaExtension({ - schema: 'prisma/schema/secondary.prisma', - version: '6.2.0', - migrate: false, -}), -``` +See the [prismaExtension documentation](/config/extensions/prismaExtension) for more information. #### syncEnvVars -The `syncEnvVars` build extension replaces the deprecated `resolveEnvVars` export. Check out our [syncEnvVars documentation](/deploy-environment-variables#sync-env-vars-from-another-service) for more information. - -```ts -import { syncEnvVars } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [syncEnvVars()], - }, -}); -``` - -#### syncVercelEnvVars - -The `syncVercelEnvVars` build extension syncs environment variables from your Vercel project to Trigger.dev. - - - You need to set the `VERCEL_ACCESS_TOKEN` and `VERCEL_PROJECT_ID` environment variables, or pass - in the token and project ID as arguments to the `syncVercelEnvVars` build extension. If you're - working with a team project, you'll also need to set `VERCEL_TEAM_ID`, which can be found in your - team settings. You can find / generate the `VERCEL_ACCESS_TOKEN` in your Vercel - [dashboard](https://vercel.com/account/settings/tokens). Make sure the scope of the token covers - the project with the environment variables you want to sync. - - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [syncVercelEnvVars()], - }, -}); -``` - -#### audioWaveform - -Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [audioWaveform()], // uses verson 1.1.0 of audiowaveform by default - }, -}); -``` +See the [syncEnvVars documentation](/config/extensions/syncEnvVars) for more information. #### puppeteer - - -To use Puppeteer in your project, add these build settings to your `trigger.config.ts` file: - -```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [puppeteer()], - }, -}); -``` - -And add the following environment variable in your Trigger.dev dashboard on the Environment Variables page: - -```bash -PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable", -``` - -Follow [this example](/guides/examples/puppeteer) to get setup with Trigger.dev and Puppeteer in your project. +See the [puppeteer documentation](/config/extensions/puppeteer) for more information. #### ffmpeg -You can add the `ffmpeg` build extension to your build process: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { ffmpeg } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ffmpeg()], - }, -}); -``` - -By default, this will install the version of `ffmpeg` that is available in the Debian package manager. If you need a specific version, you can pass in the version as an argument: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { ffmpeg } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ffmpeg({ version: "6.0-4" })], - }, -}); -``` - -This extension will also add the `FFMPEG_PATH` and `FFPROBE_PATH` to your environment variables, making it easy to use popular ffmpeg libraries like `fluent-ffmpeg`. - -Note that `fluent-ffmpeg` needs to be added to [`external`](/config/config-file#external) in your `trigger.config.ts` file. - -Follow [this example](/guides/examples/ffmpeg-video-processing) to get setup with Trigger.dev and FFmpeg in your project. +See the [ffmpeg documentation](/config/extensions/ffmpeg) for more information. #### esbuild plugins -You can easily add existing or custom esbuild plugins to your build process using the `esbuildPlugin` extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { esbuildPlugin } from "@trigger.dev/build/extensions"; -import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - esbuildPlugin( - sentryEsbuildPlugin({ - org: process.env.SENTRY_ORG, - project: process.env.SENTRY_PROJECT, - authToken: process.env.SENTRY_AUTH_TOKEN, - }), - // optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins - { placement: "last", target: "deploy" } - ), - ], - }, -}); -``` +See the [esbuild plugins documentation](/config/extensions/esbuildPlugin) for more information. #### aptGet -You can install system packages into the deployed image using using the `aptGet` extension: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { aptGet } from "@trigger.dev/build/extensions/core"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [aptGet({ packages: ["ffmpeg"] })], - }, -}); -``` - -If you want to install a specific version of a package, you can specify the version like this: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [aptGet({ packages: ["ffmpeg=6.0-4"] })], - }, -}); -``` - -#### Custom extensions - -You can create your own extensions to further customize the build process. Extensions are an object with a `name` and zero or more lifecycle hooks (`onBuildStart` and `onBuildComplete`) that allow you to modify the `BuildContext` object that is passed to the build process through adding layers. For example, this is how the `aptGet` extension is implemented: - -```ts -import { BuildExtension } from "@trigger.dev/core/v3/build"; - -export type AptGetOptions = { - packages: string[]; -}; - -export function aptGet(options: AptGetOptions): BuildExtension { - return { - name: "aptGet", - onBuildComplete(context) { - if (context.target === "dev") { - return; - } - - context.logger.debug("Adding apt-get layer", { - pkgs: options.packages, - }); - - context.addLayer({ - id: "apt-get", - image: { - pkgs: options.packages, - }, - }); - }, - }; -} -``` - -Instead of creating this function and worrying about types, you can define an extension inline in your `trigger.config.ts` file: - -```ts trigger.config.ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "", - // Your other config settings... - build: { - extensions: [ - { - name: "aptGet", - onBuildComplete(context) { - if (context.target === "dev") { - return; - } - - context.logger.debug("Adding apt-get layer", { - pkgs: ["ffmpeg"], - }); - - context.addLayer({ - id: "apt-get", - image: { - pkgs: ["ffmpeg"], - }, - }); - }, - }, - ], - }, -}); -``` - -We'll be expanding the documentation on how to create custom extensions in the future, but for now you are encouraged to look at the existing extensions in the `@trigger.dev/build` package for inspiration, which you can see in our repo [here](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/build/src/extensions) +See the [aptGet documentation](/config/extensions/aptGet) for more information. diff --git a/docs/config/extensions/additionalFiles.mdx b/docs/config/extensions/additionalFiles.mdx new file mode 100644 index 0000000000..455459a82c --- /dev/null +++ b/docs/config/extensions/additionalFiles.mdx @@ -0,0 +1,28 @@ +--- +title: "Additional Files" +sidebarTitle: "additionalFiles" +description: "Use the additionalFiles build extension to copy additional files to the build directory" +--- + +Import the `additionalFiles` build extension and use it in your `trigger.config.ts` file: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { additionalFiles } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + additionalFiles({ files: ["wrangler/wrangler.toml", "./assets/**", "./fonts/**"] }), + ], + }, +}); +``` + +This will copy the files specified in the `files` array to the build directory. The `files` array can contain globs. The output paths will match the path of the file, relative to the root of the project. + +This extension effects both the `dev` and the `deploy` commands, and the resulting paths will be the same for both. + +The root of the project is the directory that contains the trigger.config.ts file diff --git a/docs/config/extensions/additionalPackages.mdx b/docs/config/extensions/additionalPackages.mdx new file mode 100644 index 0000000000..7837c5dacd --- /dev/null +++ b/docs/config/extensions/additionalPackages.mdx @@ -0,0 +1,36 @@ +--- +title: "Additional Packages" +sidebarTitle: "additionalPackages" +description: "Use the additionalPackages build extension to include additional packages in the build" +--- + +Import the `additionalPackages` build extension and use it in your `trigger.config.ts` file: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { additionalPackages } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [additionalPackages({ packages: ["wrangler"] })], + }, +}); +``` + +This allows you to include additional packages in the build that are not automatically included via imports. This is useful if you want to install a package that includes a CLI tool that you want to invoke in your tasks via `exec`. We will try to automatically resolve the version of the package but you can specify the version by using the `@` symbol: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [additionalPackages({ packages: ["wrangler@1.19.0"] })], + }, +}); +``` + +This extension does not do anything in `dev` mode, but it will install the packages in the build directory when you run `deploy`. The packages will be installed in the `node_modules` directory in the build directory. diff --git a/docs/config/extensions/aptGet.mdx b/docs/config/extensions/aptGet.mdx new file mode 100644 index 0000000000..b45ecaa485 --- /dev/null +++ b/docs/config/extensions/aptGet.mdx @@ -0,0 +1,34 @@ +--- +title: "apt-get" +sidebarTitle: "aptGet" +description: "Use the aptGet build extension to install system packages into the deployed image" +--- + +You can install system packages into the deployed image using the `aptGet` extension: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { aptGet } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [aptGet({ packages: ["ffmpeg"] })], + }, +}); +``` + +If you want to install a specific version of a package, you can specify the version like this: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [aptGet({ packages: ["ffmpeg=6.0-4"] })], + }, +}); +``` diff --git a/docs/config/extensions/audioWaveform.mdx b/docs/config/extensions/audioWaveform.mdx new file mode 100644 index 0000000000..7b3df9f3da --- /dev/null +++ b/docs/config/extensions/audioWaveform.mdx @@ -0,0 +1,20 @@ +--- +title: "Audio Waveform" +sidebarTitle: "audioWaveform" +description: "Use the audioWaveform build extension to add support for Audio Waveform in your project" +--- + +Previously, we installed [Audio Waveform](https://github.com/bbc/audiowaveform) in the build image. That's been moved to a build extension: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { audioWaveform } from "@trigger.dev/build/extensions/audioWaveform"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [audioWaveform()], // uses verson 1.1.0 of audiowaveform by default + }, +}); +``` diff --git a/docs/config/extensions/custom.mdx b/docs/config/extensions/custom.mdx new file mode 100644 index 0000000000..2d581c6545 --- /dev/null +++ b/docs/config/extensions/custom.mdx @@ -0,0 +1,380 @@ +--- +title: "Custom build extensions" +sidebarTitle: "Custom" +description: "Customize how your project is built and deployed to Trigger.dev with your own custom build extensions" +--- + +Build extensions allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). See our [build extension overview](/config/extensions/overview) for more information on how to install and use our built-in extensions. Build extensions can do the following: + +- Add additional files to the build +- Add dependencies to the list of externals +- Add esbuild plugins +- Add additional npm dependencies +- Add additional system packages to the image build container +- Add commands to run in the image build container +- Add environment variables to the image build container +- Sync environment variables to your Trigger.dev project + +## Creating a build extension + +Build extensions are added to your `trigger.config.ts` file, with a required `name` and optional build hook functions. Here's a simple example of a build extension that just logs a message when the build starts: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [ + { + name: "my-extension", + onBuildStart: async (context) => { + console.log("Build starting!"); + }, + }, + ], + }, +}); +``` + +You can also extract that out into a function instead of defining it inline, in which case you will need to import the `BuildExtension` type from the `@trigger.dev/build` package: + + + You'll need to add the `@trigger.dev/build` package to your `devDependencies` before the below + code will work. Make sure it's version matches that of the installed `@trigger.dev/sdk` package. + + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { BuildExtension } from "@trigger.dev/build"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [myExtension()], + }, +}); + +function myExtension(): BuildExtension { + return { + name: "my-extension", + onBuildStart: async (context) => { + console.log("Build starting!"); + }, + }; +} +``` + +## Build hooks + +### externalsForTarget + +This allows the extension to add additional dependencies to the list of externals for the build. This is useful for dependencies that are not included in the bundle, but are expected to be available at runtime. + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [ + { + name: "my-extension", + externalsForTarget: async (target) => { + return ["my-dependency"]; + }, + }, + ], + }, +}); +``` + +### onBuildStart + +This hook runs before the build starts. It receives the `BuildContext` object as an argument. + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [ + { + name: "my-extension", + onBuildStart: async (context) => { + console.log("Build starting!"); + }, + }, + ], + }, +}); +``` + +If you want to add an esbuild plugin, you must do so in the `onBuildStart` hook. Here's an example of adding a custom esbuild plugin: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [ + { + name: "my-extension", + onBuildStart: async (context) => { + context.registerPlugin({ + name: "my-plugin", + setup(build) { + build.onLoad({ filter: /.*/, namespace: "file" }, async (args) => { + return { + contents: "console.log('Hello, world!')", + loader: "js", + }; + }); + }, + }); + }, + }, + ], + }, +}); +``` + +You can use the `BuildContext.target` property to determine if the build is for `dev` or `deploy`: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [ + { + name: "my-extension", + onBuildStart: async (context) => { + if (context.target === "dev") { + console.log("Building for dev"); + } else { + console.log("Building for deploy"); + } + }, + }, + ], + }, +}); +``` + +### onBuildComplete + +This hook runs after the build completes. It receives the `BuildContext` object and a `BuildManifest` object as arguments. This is where you can add in one or more `BuildLayer`'s to the context. + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "my-project", + build: { + extensions: [ + { + name: "my-extension", + onBuildComplete: async (context, manifest) => { + context.addLayer({ + id: "more-dependencies", + dependencies, + }); + }, + }, + ], + }, +}); +``` + +See the [addLayer](#addlayer) documentation for more information on how to use `addLayer`. + +## BuildTarget + +Can either be `dev` or `deploy`, matching the CLI command name that is being run. + +```sh +npx trigger.dev@latest dev # BuildTarget is "dev" +npx trigger.dev@latest deploy # BuildTarget is "deploy" +``` + +## BuildContext + +### addLayer() + + + The layer to add to the build context. See the [BuildLayer](#buildlayer) documentation for more + information. + + +### registerPlugin() + + + The esbuild plugin to register. + + + + + + An optional target to register the plugin for. If not provided, the plugin will be registered + for all targets. + + + An optional placement for the plugin. If not provided, the plugin will be registered in place. + This allows you to control the order of plugins. + + + + +### resolvePath() + +Resolves a path relative to the project's working directory. + + + The path to resolve. + + +```ts +const resolvedPath = context.resolvePath("my-other-dependency"); +``` + +### properties + + + The target of the build, either `dev` or `deploy`. + + + + + + The runtime of the project (either node or bun) + + + The project ref + + + The trigger directories to search for tasks + + + The build configuration object + + + The working directory of the project + + + The root workspace directory of the project + + + The path to the package.json file + + + The path to the lockfile (package-lock.json, yarn.lock, or pnpm-lock.yaml) + + + The path to the trigger.config.ts file + + + The path to the tsconfig.json file + + + + + + A logger object that can be used to log messages to the console. + + +## BuildLayer + + + A unique identifier for the layer. + + + + An array of commands to run in the image build container. + +```ts +commands: ["echo 'Hello, world!'"]; +``` + +These commands are run after packages have been installed and the code copied into the container in the "build" stage of the Dockerfile. This means you cannot install system packages in these commands because they won't be available in the final stage. To do that, please use the `pkgs` property of the `image` object. + + + + + + + An array of system packages to install in the image build container. + + + An array of instructions to add to the Dockerfile. + + + + + + + + Environment variables to add to the image build container, but only during the "build" stage + of the Dockerfile. This is where you'd put environment variables that are needed when running + any of the commands in the `commands` array. + + + + + + + + Environment variables that should sync to the Trigger.dev project, which will then be avalable + in your tasks at runtime. Importantly, these are NOT added to the image build container, but + are instead added to the Trigger.dev project and stored securely. + + + + + + An object of dependencies to add to the build. The key is the package name and the value is the + version. + +```ts +dependencies: { + "my-dependency": "^1.0.0", +}; +``` + + + +### examples + +Add a command that will echo the value of an environment variable: + +```ts +context.addLayer({ + id: "my-layer", + commands: [`echo $MY_ENV_VAR`], + build: { + env: { + MY_ENV_VAR: "Hello, world!", + }, + }, +}); +``` + +## Troubleshooting + +When creating a build extension, you may run into issues with the build process. One thing that can help is turning on `debug` logging when running either `dev` or `deploy`: + +```sh +npx trigger.dev@latest dev --log-level debug +npx trigger.dev@latest deploy --log-level debug +``` + +Another helpful tool is the `--dry-run` flag on the `deploy` command, which will bundle your project and generate the Containerfile (e.g. the Dockerfile) without actually deploying it. This can help you see what the final image will look like and debug any issues with the build process. + +```sh +npx trigger.dev@latest deploy --dry-run +``` + +You should also take a look at our built in extensions for inspiration on how to create your own. You can find them in in [the source code here](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/build/src/extensions). diff --git a/docs/config/extensions/emitDecoratorMetadata.mdx b/docs/config/extensions/emitDecoratorMetadata.mdx new file mode 100644 index 0000000000..49f5d399b7 --- /dev/null +++ b/docs/config/extensions/emitDecoratorMetadata.mdx @@ -0,0 +1,29 @@ +--- +title: "Emit Decorator Metadata" +sidebarTitle: "emitDecoratorMetadata" +description: "Use the emitDecoratorMetadata build extension to enable support for the emitDecoratorMetadata TypeScript compiler option" +--- + +If you need support for the `emitDecoratorMetadata` typescript compiler option, import the `emitDecoratorMetadata` build extension and use it in your `trigger.config.ts` file: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [emitDecoratorMetadata()], + }, +}); +``` + +This is usually required if you are using certain ORMs, like TypeORM, that require this option to be enabled. It's not enabled by default because there is a performance cost to enabling it. + + + emitDecoratorMetadata works by hooking into the esbuild bundle process and using the TypeScript + compiler API to compile files where we detect the use of decorators. This means you must have + `emitDecoratorMetadata` enabled in your `tsconfig.json` file, as well as `typescript` installed in + your `devDependencies`. + diff --git a/docs/config/extensions/esbuildPlugin.mdx b/docs/config/extensions/esbuildPlugin.mdx new file mode 100644 index 0000000000..bc0a7fc87b --- /dev/null +++ b/docs/config/extensions/esbuildPlugin.mdx @@ -0,0 +1,31 @@ +--- +title: "esbuild Plugin" +sidebarTitle: "esbuildPlugin" +description: "Use the esbuildPlugin build extension to add existing or custom esbuild plugins to your build process" +--- + +You can easily add existing or custom esbuild plugins to your build process using the `esbuildPlugin` extension: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { esbuildPlugin } from "@trigger.dev/build/extensions"; +import { sentryEsbuildPlugin } from "@sentry/esbuild-plugin"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + esbuildPlugin( + sentryEsbuildPlugin({ + org: process.env.SENTRY_ORG, + project: process.env.SENTRY_PROJECT, + authToken: process.env.SENTRY_AUTH_TOKEN, + }), + // optional - only runs during the deploy command, and adds the plugin to the end of the list of plugins + { placement: "last", target: "deploy" } + ), + ], + }, +}); +``` diff --git a/docs/config/extensions/ffmpeg.mdx b/docs/config/extensions/ffmpeg.mdx new file mode 100644 index 0000000000..390e2796f0 --- /dev/null +++ b/docs/config/extensions/ffmpeg.mdx @@ -0,0 +1,41 @@ +--- +title: "FFmpeg" +sidebarTitle: "ffmpeg" +description: "Use the ffmpeg build extension to include FFmpeg in your project" +--- + +You can add the `ffmpeg` build extension to your build process: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { ffmpeg } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ffmpeg()], + }, +}); +``` + +By default, this will install the version of `ffmpeg` that is available in the Debian package manager. If you need a specific version, you can pass in the version as an argument: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { ffmpeg } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ffmpeg({ version: "6.0-4" })], + }, +}); +``` + +This extension will also add the `FFMPEG_PATH` and `FFPROBE_PATH` to your environment variables, making it easy to use popular ffmpeg libraries like `fluent-ffmpeg`. + +Note that `fluent-ffmpeg` needs to be added to [`external`](/config/config-file#external) in your `trigger.config.ts` file. + +Follow [this example](/guides/examples/ffmpeg-video-processing) to get setup with Trigger.dev and FFmpeg in your project. diff --git a/docs/config/extensions/overview.mdx b/docs/config/extensions/overview.mdx index 12da54d069..412a11062b 100644 --- a/docs/config/extensions/overview.mdx +++ b/docs/config/extensions/overview.mdx @@ -1,96 +1,14 @@ --- title: "Build extensions" +sidebarTitle: "Overview" description: "Customize how your project is built and deployed to Trigger.dev with build extensions" --- -Build extension allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). See our [trigger.config.ts reference](/config/config-file#extensions) for more information on how to install and use our built-in extensions. Build extensions can do the following: +Build extensions allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). -- Add additional files to the build -- Add dependencies to the list of externals -- Add esbuild plugins -- Add additional npm dependencies -- Add additional system packages to the image build container -- Add commands to run in the image build container -- Add environment variables to the image build container -- Sync environment variables to your Trigger.dev project +You can use pre-built extensions by installing the `@trigger.dev/build` package into your `devDependencies`, or you can create your own. -## Creating a build extension - -Build extensions are added to your `trigger.config.ts` file, with a required `name` and optional build hook functions. Here's a simple example of a build extension that just logs a message when the build starts: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "my-project", - build: { - extensions: [ - { - name: "my-extension", - onBuildStart: async (context) => { - console.log("Build starting!"); - }, - }, - ], - }, -}); -``` - -You can also extract that out into a function instead of defining it inline, in which case you will need to import the `BuildExtension` type from the `@trigger.dev/build` package: - - - You'll need to add the `@trigger.dev/build` package to your `devDependencies` before the below - code will work. Make sure it's version matches that of the installed `@trigger.dev/sdk` package. - - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; -import { BuildExtension } from "@trigger.dev/build"; - -export default defineConfig({ - project: "my-project", - build: { - extensions: [myExtension()], - }, -}); - -function myExtension(): BuildExtension { - return { - name: "my-extension", - onBuildStart: async (context) => { - console.log("Build starting!"); - }, - }; -} -``` - -## Build hooks - -### externalsForTarget - -This allows the extension to add additional dependencies to the list of externals for the build. This is useful for dependencies that are not included in the bundle, but are expected to be available at runtime. - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "my-project", - build: { - extensions: [ - { - name: "my-extension", - externalsForTarget: async (target) => { - return ["my-dependency"]; - }, - }, - ], - }, -}); -``` - -### onBuildStart - -This hook runs before the build starts. It receives the `BuildContext` object as an argument. +Build extensions are added to your `trigger.config.ts` file under the `build.extensions` property: ```ts import { defineConfig } from "@trigger.dev/sdk/v3"; @@ -110,270 +28,39 @@ export default defineConfig({ }); ``` -If you want to add an esbuild plugin, you must do so in the `onBuildStart` hook. Here's an example of adding a custom esbuild plugin: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "my-project", - build: { - extensions: [ - { - name: "my-extension", - onBuildStart: async (context) => { - context.registerPlugin({ - name: "my-plugin", - setup(build) { - build.onLoad({ filter: /.*/, namespace: "file" }, async (args) => { - return { - contents: "console.log('Hello, world!')", - loader: "js", - }; - }); - }, - }); - }, - }, - ], - }, -}); -``` - -You can use the `BuildContext.target` property to determine if the build is for `dev` or `deploy`: - -```ts -import { defineConfig } from "@trigger.dev/sdk/v3"; - -export default defineConfig({ - project: "my-project", - build: { - extensions: [ - { - name: "my-extension", - onBuildStart: async (context) => { - if (context.target === "dev") { - console.log("Building for dev"); - } else { - console.log("Building for deploy"); - } - }, - }, - ], - }, -}); -``` - -### onBuildComplete - -This hook runs after the build completes. It receives the `BuildContext` object and a `BuildManifest` object as arguments. This is where you can add in one or more `BuildLayer`'s to the context. +If you are using a pre-built extension, you can import it from the `@trigger.dev/build` package: ```ts import { defineConfig } from "@trigger.dev/sdk/v3"; +import { ffmpeg } from "@trigger.dev/build/extensions/core"; export default defineConfig({ project: "my-project", build: { - extensions: [ - { - name: "my-extension", - onBuildComplete: async (context, manifest) => { - context.addLayer({ - id: "more-dependencies", - dependencies, - }); - }, - }, - ], - }, -}); -``` - -See the [addLayer](#addlayer) documentation for more information on how to use `addLayer`. - -## BuildTarget - -Can either be `dev` or `deploy`, matching the CLI command name that is being run. - -```sh -npx trigger.dev@latest dev # BuildTarget is "dev" -npx trigger.dev@latest deploy # BuildTarget is "deploy" -``` - -## BuildContext - -### addLayer() - - - The layer to add to the build context. See the [BuildLayer](#buildlayer) documentation for more - information. - - -### registerPlugin() - - - The esbuild plugin to register. - - - - - - An optional target to register the plugin for. If not provided, the plugin will be registered - for all targets. - - - An optional placement for the plugin. If not provided, the plugin will be registered in place. - This allows you to control the order of plugins. - - - - -### resolvePath() - -Resolves a path relative to the project's working directory. - - - The path to resolve. - - -```ts -const resolvedPath = context.resolvePath("my-other-dependency"); -``` - -### properties - - - The target of the build, either `dev` or `deploy`. - - - - - - The runtime of the project (either node or bun) - - - The project ref - - - The trigger directories to search for tasks - - - The build configuration object - - - The working directory of the project - - - The root workspace directory of the project - - - The path to the package.json file - - - The path to the lockfile (package-lock.json, yarn.lock, or pnpm-lock.yaml) - - - The path to the trigger.config.ts file - - - The path to the tsconfig.json file - - - - - - A logger object that can be used to log messages to the console. - - -## BuildLayer - - - A unique identifier for the layer. - - - - An array of commands to run in the image build container. - -```ts -commands: ["echo 'Hello, world!'"]; -``` - -These commands are run after packages have been installed and the code copied into the container in the "build" stage of the Dockerfile. This means you cannot install system packages in these commands because they won't be available in the final stage. To do that, please use the `pkgs` property of the `image` object. - - - - - - - An array of system packages to install in the image build container. - - - An array of instructions to add to the Dockerfile. - - - - - - - - Environment variables to add to the image build container, but only during the "build" stage - of the Dockerfile. This is where you'd put environment variables that are needed when running - any of the commands in the `commands` array. - - - - - - - - Environment variables that should sync to the Trigger.dev project, which will then be avalable - in your tasks at runtime. Importantly, these are NOT added to the image build container, but - are instead added to the Trigger.dev project and stored securely. - - - - - - An object of dependencies to add to the build. The key is the package name and the value is the - version. - -```ts -dependencies: { - "my-dependency": "^1.0.0", -}; -``` - - - -### examples - -Add a command that will echo the value of an environment variable: - -```ts -context.addLayer({ - id: "my-layer", - commands: [`echo $MY_ENV_VAR`], - build: { - env: { - MY_ENV_VAR: "Hello, world!", - }, + extensions: [ffmpeg()], }, }); ``` -## Troubleshooting - -When creating a build extension, you may run into issues with the build process. One thing that can help is turning on `debug` logging when running either `dev` or `deploy`: +## Built-in extensions -```sh -npx trigger.dev@latest dev --log-level debug -npx trigger.dev@latest deploy --log-level debug -``` +Trigger.dev provides a set of built-in extensions that you can use to customize how your project is built and deployed. These extensions are available out of the box and can be configured in your `trigger.config.ts` file. -Another helpful tool is the `--dry-run` flag on the `deploy` command, which will bundle your project and generate the Containerfile (e.g. the Dockerfile) without actually deploying it. This can help you see what the final image will look like and debug any issues with the build process. +| Extension | Description | +| :-------------------------------------------------------------------- | :----------------------------------------------------------------------------- | +| [prismaExtension](/config/extensions/prismaExtension) | Using prisma in your Trigger.dev tasks | +| [pythonExtension](/config/extensions/pythonExtension) | Execute Python scripts in your project | +| [puppeteer](/config/extensions/puppeteer) | Use Puppeteer in your Trigger.dev tasks | +| [ffmpeg](/config/extensions/ffmpeg) | Use FFmpeg in your Trigger.dev tasks | +| [aptGet](/config/extensions/aptGet) | Install system packages in your build image | +| [additionalFiles](/config/extensions/additionalFiles) | Copy additional files to your build image | +| [additionalPackages](/config/extensions/additionalPackages) | Install additional npm packages in your build image | +| [syncEnvVars](/config/extensions/syncEnvVars) | Automatically sync environment variables from external services to Trigger.dev | +| [syncVercelEnvVars](/config/extensions/syncEnvVars#syncVercelEnvVars) | Automatically sync environment variables from Vercel to Trigger.dev | +| [esbuildPlugin](/config/extensions/esbuildPlugin) | Add existing or custom esbuild extensions to customize your build process | +| [emitDecoratorMetadata](/config/extensions/emitDecoratorMetadata) | Enable `emitDecoratorMetadata` in your TypeScript build | +| [audioWaveform](/config/extensions/audioWaveform) | Add Audio Waveform to your build image | -```sh -npx trigger.dev@latest deploy --dry-run -``` +## Custom extensions -You should also take a look at our built in extensions for inspiration on how to create your own. You can find them in in [the source code here](https://github.com/triggerdotdev/trigger.dev/tree/main/packages/build/src/extensions). +If one of the built-in extensions doesn't meet your needs, you can create your own custom extension. See our [guide on creating custom build extensions](/config/extensions/custom) for more information. diff --git a/docs/config/extensions/prismaExtension.mdx b/docs/config/extensions/prismaExtension.mdx new file mode 100644 index 0000000000..73bc94b16a --- /dev/null +++ b/docs/config/extensions/prismaExtension.mdx @@ -0,0 +1,157 @@ +--- +title: "Prisma" +sidebarTitle: "prismaExtension" +description: "Use the prismaExtension build extension to use Prisma with Trigger.dev" +--- + +If you are using Prisma, you should use the prisma build extension. + +- Automatically handles copying Prisma files to the build directory +- Generates the Prisma client during the deploy process +- Optionally will migrate the database during the deploy process +- Support for TypedSQL and multiple schema files +- You can use `prismaSchemaFolder` to specify just the directory containing your schema file, instead of the full path +- You can add the extension twice if you have multiple separate schemas in the same project (example below) + +You can use it for a simple Prisma setup like this: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + prismaExtension({ + version: "5.19.0", // optional, we'll automatically detect the version if not provided + schema: "prisma/schema.prisma", + }), + ], + }, +}); +``` + + + This does not have any effect when running the `dev` command, only when running the `deploy` + command. + + +### Migrations + +If you want to also run migrations during the build process, you can pass in the `migrate` option: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + prismaExtension({ + schema: "prisma/schema.prisma", + migrate: true, + directUrlEnvVarName: "DATABASE_URL_UNPOOLED", // optional - the name of the environment variable that contains the direct database URL if you are using a direct database URL + }), + ], + }, +}); +``` + +### clientGenerator + +If you have multiple `generator` statements defined in your schema file, you can pass in the `clientGenerator` option to specify the `prisma-client-js` generator, which will prevent other generators from being generated. Some examples where you may need to do this include when using the `prisma-kysely` or `prisma-json-types-generator` generators. + + + +```prisma schema.prisma +datasource db { + provider = "postgresql" + url = env("DATABASE_URL") + directUrl = env("DATABASE_URL_UNPOOLED") +} + +// We only want to generate the prisma-client-js generator +generator client { + provider = "prisma-client-js" +} + +generator kysely { + provider = "prisma-kysely" + output = "../../src/kysely" + enumFileName = "enums.ts" + fileName = "types.ts" +} + +generator json { + provider = "prisma-json-types-generator" +} +``` + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + prismaExtension({ + schema: "prisma/schema.prisma", + clientGenerator: "client", + }), + ], + }, +}); +``` + + + +### TypedSQL + +If you are using [TypedSQL](https://www.prisma.io/typedsql), you'll need to enable it via the `typedSql` option: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + prismaExtension({ + schema: "prisma/schema.prisma", + typedSql: true, + }), + ], + }, +}); +``` + + + The `prismaExtension` will inject the `DATABASE_URL` environment variable into the build process. Learn more about setting environment variables for deploying in our [Environment Variables](/deploy-environment-variables) guide. + +These environment variables are only used during the build process and are not embedded in the final container image. + + + +### Multiple schemas + +If you have multiple separate schemas in the same project you can add the extension multiple times: + +```ts +prismaExtension({ + schema: 'prisma/schema/main.prisma', + version: '6.2.0', + migrate: false, +}), +prismaExtension({ + schema: 'prisma/schema/secondary.prisma', + version: '6.2.0', + migrate: false, +}), +``` diff --git a/docs/config/extensions/puppeteer.mdx b/docs/config/extensions/puppeteer.mdx new file mode 100644 index 0000000000..62942e7f87 --- /dev/null +++ b/docs/config/extensions/puppeteer.mdx @@ -0,0 +1,30 @@ +--- +title: "Puppeteer" +sidebarTitle: "puppeteer" +description: "Use the puppeteer build extension to enable support for Puppeteer in your project" +--- + + + +To use Puppeteer in your project, add these build settings to your `trigger.config.ts` file: + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [puppeteer()], + }, +}); +``` + +And add the following environment variable in your Trigger.dev dashboard on the Environment Variables page: + +```bash +PUPPETEER_EXECUTABLE_PATH: "/usr/bin/google-chrome-stable", +``` + +Follow [this example](/guides/examples/puppeteer) to get setup with Trigger.dev and Puppeteer in your project. diff --git a/docs/config/extensions/pythonExtension.mdx b/docs/config/extensions/pythonExtension.mdx new file mode 100644 index 0000000000..21704d65d3 --- /dev/null +++ b/docs/config/extensions/pythonExtension.mdx @@ -0,0 +1,182 @@ +--- +title: "Python" +sidebarTitle: "pythonExtension" +description: "Use the python build extension to add support for executing Python scripts in your project" +--- + +If you need to execute Python scripts in your Trigger.dev project, you can use the `pythonExtension` build extension via the `@trigger.dev/python` package. + +First, you'll need to install the `@trigger.dev/python` package: + +```bash +npm add @trigger.dev/python +``` + +Then, you can use the `pythonExtension` build extension in your `trigger.config.ts` file: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { pythonExtension } from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [pythonExtension()], + }, +}); +``` + +This will take care of adding python to the build image and setting up the necessary environment variables to execute Python scripts. You can then use our `python` utilities in the `@trigger.dev/python` package to execute Python scripts in your tasks. For example, running a Python script inline in a task: + +```ts +import { task } from "@trigger.dev/sdk/v3"; +import { python } from "@trigger.dev/python"; + +export const myScript = task({ + id: "my-python-script", + run: async () => { + const result = await python.runInline(`print("Hello, world!")`); + return result.stdout; + }, +}); +``` + +## Adding python scripts + +You can automatically add python scripts to your project using the `scripts` option in the `pythonExtension` function. This will copy the specified scripts to the build directory during the deploy process. For example: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { pythonExtension } from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + pythonExtension({ + scripts: ["./python/**/*.py"], + }), + ], + }, +}); +``` + +This will copy all Python files in the `python` directory to the build directory during the deploy process. You can then execute these scripts using the `python.runScript` function: + +```ts +import { task } from "@trigger.dev/sdk/v3"; +import { python } from "@trigger.dev/python"; + +export const myScript = task({ + id: "my-python-script", + run: async () => { + const result = await python.runScript("./python/my_script.py", ["hello", "world"]); + return result.stdout; + }, +}); +``` + + + The pythonExtension will also take care of moving the scripts to the correct location during `dev` + mode, so you can use the same exact path in development as you do in production. + + +## Using requirements files + +If you have a `requirements.txt` file in your project, you can use the `requirementsFile` option in the `pythonExtension` function to install the required packages during the build process. For example: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { pythonExtension } from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + pythonExtension({ + requirementsFile: "./requirements.txt", + }), + ], + }, +}); +``` + +This will install the packages specified in the `requirements.txt` file during the build process. You can then use these packages in your Python scripts. + + + The `requirementsFile` option is only available in production mode. In development mode, you can + install the required packages manually using the `pip` command. + + +## Virtual environments + +If you are using a virtual environment in your project, you can use the `devPythonBinaryPath` option in the `pythonExtension` function to specify the path to the Python binary in the virtual environment. For example: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { pythonExtension } from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + pythonExtension({ + devPythonBinaryPath: ".venv/bin/python", + }), + ], + }, +}); +``` + +This has no effect in production mode, but in development mode, it will use the specified Python binary to execute Python scripts. + +## Streaming output + +All of the `python` functions have a streaming version that allows you to stream the output of the Python script as it runs. For example: + +```ts +import { task } from "@trigger.dev/sdk/v3"; +import { python } from "@trigger.dev/python"; + +export const myStreamingScript = task({ + id: "my-streaming-python-script", + run: async () => { + // You don't need to await the result + const result = python.stream.runScript("./python/my_script.py", ["hello", "world"]); + + // result is an async iterable/readable stream + for await (const chunk of streamingResult) { + console.log(chunk); + } + }, +}); +``` + +## Environment variables + +We automatically inject the environment variables in the `process.env` object when running Python scripts. You can access these environment variables in your Python scripts using the `os.environ` dictionary. For example: + +```python +import os + +print(os.environ["MY_ENV_VAR"]) +``` + +You can also pass additional environment variables to the Python script using the `env` option in the `python.runScript` function. For example: + +```ts +import { task } from "@trigger.dev/sdk/v3"; +import { python } from "@trigger.dev/python"; + +export const myScript = task({ + id: "my-python-script", + run: async () => { + const result = await python.runScript("./python/my_script.py", ["hello", "world"], { + env: { + MY_ENV_VAR: "my value", + }, + }); + return result.stdout; + }, +}); +``` diff --git a/docs/config/extensions/syncEnvVars.mdx b/docs/config/extensions/syncEnvVars.mdx new file mode 100644 index 0000000000..c4fc338fec --- /dev/null +++ b/docs/config/extensions/syncEnvVars.mdx @@ -0,0 +1,116 @@ +--- +title: "Sync env vars" +sidebarTitle: "syncEnvVars" +description: "Use the syncEnvVars build extension to automatically sync environment variables to Trigger.dev" +--- + +The `syncEnvVars` build extension will sync env vars from another service into Trigger.dev before the deployment starts. This is useful if you are using a secret store service like Infisical or AWS Secrets Manager to store your secrets. + +`syncEnvVars` takes an async callback function, and any env vars returned from the callback will be synced to Trigger.dev. + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { syncEnvVars } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + build: { + extensions: [ + syncEnvVars(async (ctx) => { + return [ + { name: "SECRET_KEY", value: "secret-value" }, + { name: "ANOTHER_SECRET", value: "another-secret-value" }, + ]; + }), + ], + }, +}); +``` + +The callback is passed a context object with the following properties: + +- `environment`: The environment name that the task is being deployed to (e.g. `production`, `staging`, etc.) +- `projectRef`: The project ref of the Trigger.dev project +- `env`: The environment variables that are currently set in the Trigger.dev project + +### Example: Sync env vars from Infisical + +In this example we're using env vars from [Infisical](https://infisical.com). + +```ts trigger.config.ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { syncEnvVars } from "@trigger.dev/build/extensions/core"; +import { InfisicalSDK } from "@infisical/sdk"; + +export default defineConfig({ + build: { + extensions: [ + syncEnvVars(async (ctx) => { + const client = new InfisicalSDK(); + + await client.auth().universalAuth.login({ + clientId: process.env.INFISICAL_CLIENT_ID!, + clientSecret: process.env.INFISICAL_CLIENT_SECRET!, + }); + + const { secrets } = await client.secrets().listSecrets({ + environment: ctx.environment, + projectId: process.env.INFISICAL_PROJECT_ID!, + }); + + return secrets.map((secret) => ({ + name: secret.secretKey, + value: secret.secretValue, + })); + }), + ], + }, +}); +``` + +### syncVercelEnvVars + +The `syncVercelEnvVars` build extension syncs environment variables from your Vercel project to Trigger.dev. + + + You need to set the `VERCEL_ACCESS_TOKEN` and `VERCEL_PROJECT_ID` environment variables, or pass + in the token and project ID as arguments to the `syncVercelEnvVars` build extension. If you're + working with a team project, you'll also need to set `VERCEL_TEAM_ID`, which can be found in your + team settings. You can find / generate the `VERCEL_ACCESS_TOKEN` in your Vercel + [dashboard](https://vercel.com/account/settings/tokens). Make sure the scope of the token covers + the project with the environment variables you want to sync. + + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + // This will automatically use the VERCEL_ACCESS_TOKEN and VERCEL_PROJECT_ID environment variables + extensions: [syncVercelEnvVars()], + }, +}); +``` + +Or you can pass in the token and project ID as arguments: + +```ts +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { syncVercelEnvVars } from "@trigger.dev/build/extensions/core"; + +export default defineConfig({ + project: "", + // Your other config settings... + build: { + extensions: [ + syncVercelEnvVars({ + projectId: "your-vercel-project-id", + vercelAccessToken: "your-vercel-access-token", + vercelTeamId: "your-vercel-team-id", // optional + }), + ], + }, +}); +``` diff --git a/docs/docs.json b/docs/docs.json index ebbf443f7f..6fd9790dc7 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -39,36 +39,7 @@ }, "triggering", "runs", - "apikeys", - { - "group": "Configuration", - "pages": [ - "config/config-file", - "config/extensions/overview" - ] - } - ] - }, - { - "group": "Development", - "pages": [ - "cli-dev", - "run-tests" - ] - }, - { - "group": "Deployment", - "pages": [ - "deployment/overview", - "deploy-environment-variables", - "github-actions", - "deployment/atomic-deployment", - { - "group": "Deployment integrations", - "pages": [ - "vercel-integration" - ] - } + "apikeys" ] }, { @@ -91,14 +62,61 @@ "versioning", "machines", "idempotency", - "replaying", "runs/max-duration", "tags", "runs/metadata", "run-usage", - "context", - "bulk-actions", - "examples" + "context" + ] + }, + { + "group": "Configuration", + "pages": [ + "config/config-file", + { + "group": "Build extensions", + "pages": [ + "config/extensions/overview", + { + "group": "Built-in extensions", + "pages": [ + "config/extensions/prismaExtension", + "config/extensions/pythonExtension", + "config/extensions/puppeteer", + "config/extensions/ffmpeg", + "config/extensions/aptGet", + "config/extensions/additionalFiles", + "config/extensions/additionalPackages", + "config/extensions/syncEnvVars", + "config/extensions/esbuildPlugin", + "config/extensions/emitDecoratorMetadata", + "config/extensions/audioWaveform" + ] + }, + "config/extensions/custom" + ] + } + ] + }, + { + "group": "Development", + "pages": [ + "cli-dev" + ] + }, + { + "group": "Deployment", + "pages": [ + "deployment/overview", + "deploy-environment-variables", + "github-actions", + "deployment/atomic-deployment", + { + "group": "Deployment integrations", + "pages": [ + "vercel-integration" + ] + } ] }, { @@ -146,6 +164,15 @@ } ] }, + { + "group": "Using the Dashboard", + "pages": [ + "run-tests", + "troubleshooting-alerts", + "replaying", + "bulk-actions" + ] + }, { "group": "Troubleshooting", "pages": [ @@ -153,7 +180,6 @@ "troubleshooting-debugging-in-vscode", "upgrading-packages", "upgrading-beta", - "troubleshooting-alerts", "troubleshooting-uptime-status", "troubleshooting-github-issues", "request-feature" diff --git a/docs/guides/examples/ffmpeg-video-processing.mdx b/docs/guides/examples/ffmpeg-video-processing.mdx index fa18b2d323..145545311b 100644 --- a/docs/guides/examples/ffmpeg-video-processing.mdx +++ b/docs/guides/examples/ffmpeg-video-processing.mdx @@ -29,7 +29,7 @@ export default defineConfig({ ``` - [Build extensions](/config/config-file#extensions) allow you to hook into the build system and + [Build extensions](/config/extensions/overview) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/guides/examples/libreoffice-pdf-conversion.mdx b/docs/guides/examples/libreoffice-pdf-conversion.mdx index 50f4299f68..35d821834e 100644 --- a/docs/guides/examples/libreoffice-pdf-conversion.mdx +++ b/docs/guides/examples/libreoffice-pdf-conversion.mdx @@ -34,7 +34,7 @@ export default defineConfig({ ``` - [Build extensions](/config/config-file#extensions) allow you to hook into the build system and + [Build extensions](/config/extensions/overview) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/guides/examples/pdf-to-image.mdx b/docs/guides/examples/pdf-to-image.mdx index 7caa58fdd8..4c85264577 100644 --- a/docs/guides/examples/pdf-to-image.mdx +++ b/docs/guides/examples/pdf-to-image.mdx @@ -12,7 +12,7 @@ This example demonstrates how to use Trigger.dev to turn a PDF into a series of ## Update your build configuration -To use this example, add these build settings below to your `trigger.config.ts` file. They ensure that the `mutool` and `curl` packages are installed when you deploy your task. You can learn more about this and see more build settings [here](/config/config-file#aptget). +To use this example, add these build settings below to your `trigger.config.ts` file. They ensure that the `mutool` and `curl` packages are installed when you deploy your task. You can learn more about this and see more build settings [here](/config/extensions/aptGet). ```ts trigger.config.ts export default defineConfig({ @@ -100,5 +100,3 @@ To test this task in the dashboard, you can use the following payload: ``` - - diff --git a/docs/guides/examples/puppeteer.mdx b/docs/guides/examples/puppeteer.mdx index b51827a82d..63e802aa02 100644 --- a/docs/guides/examples/puppeteer.mdx +++ b/docs/guides/examples/puppeteer.mdx @@ -22,7 +22,7 @@ There are 3 example tasks to follow on this page: -## Build configurations +## Build configuration To use all examples on this page, you'll first need to add these build settings to your `trigger.config.ts` file: @@ -40,7 +40,7 @@ export default defineConfig({ }); ``` -Learn more about [build configurations](/config/config-file#build-configuration) including setting default retry settings, customizing the build environment, and more. +Learn more about the [trigger.config.ts](/config/config-file) file including setting default retry settings, customizing the build environment, and more. ## Set an environment variable diff --git a/docs/guides/examples/scrape-hacker-news.mdx b/docs/guides/examples/scrape-hacker-news.mdx index dc17bc4808..69626ba1fc 100644 --- a/docs/guides/examples/scrape-hacker-news.mdx +++ b/docs/guides/examples/scrape-hacker-news.mdx @@ -12,9 +12,9 @@ import ScrapingWarning from "/snippets/web-scraping-warning.mdx"; height="315" src="https://www.youtube.com/embed/6azvzrZITKY?si=muKtsBiS9TJGGKWg" title="YouTube video player" - allow="accelerometer; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" - referrerPolicy="strict-origin-when-cross-origin" - allowFullScreen + allow="accelerometer; clipboard-write; encrypted-media; gyroscope; picture-in-picture; web-share" + referrerPolicy="strict-origin-when-cross-origin" + allowFullScreen /> ## Overview @@ -63,7 +63,7 @@ export default defineConfig({ }); ``` -Learn more about [build configurations](/config/config-file#build-configuration) including setting default retry settings, customizing the build environment, and more. +Learn more about the [trigger.config.ts](/config/config-file) file including setting default retry settings, customizing the build environment, and more. ### Environment variables diff --git a/docs/guides/examples/sentry-error-tracking.mdx b/docs/guides/examples/sentry-error-tracking.mdx index f975c2a4e9..136157cb9e 100644 --- a/docs/guides/examples/sentry-error-tracking.mdx +++ b/docs/guides/examples/sentry-error-tracking.mdx @@ -67,7 +67,7 @@ export default defineConfig({ ``` - [Build extensions](/config/config-file#extensions) allow you to hook into the build system and + [Build extensions](/config/extensions/overview) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/guides/examples/vercel-sync-env-vars.mdx b/docs/guides/examples/vercel-sync-env-vars.mdx index bdf83afd12..37a265d075 100644 --- a/docs/guides/examples/vercel-sync-env-vars.mdx +++ b/docs/guides/examples/vercel-sync-env-vars.mdx @@ -34,7 +34,7 @@ export default defineConfig({ ``` - [Build extensions](/config/config-file#extensions) allow you to hook into the build system and + [Build extensions](/config/extensions/overview) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/guides/frameworks/prisma.mdx b/docs/guides/frameworks/prisma.mdx index 83cad0299d..b85c33b70c 100644 --- a/docs/guides/frameworks/prisma.mdx +++ b/docs/guides/frameworks/prisma.mdx @@ -84,7 +84,7 @@ Next, configure the Prisma [build extension](https://trigger.dev/docs/config/ext This will ensure that the Prisma client is available when the task runs. -For a full list of options available in the Prisma build extension, see the [Prisma build extension documentation](https://trigger.dev/docs/config/config-file#prisma). +For a full list of options available in the Prisma build extension, see the [Prisma build extension documentation](https://trigger.dev/docs/config/extensions/prismaExtension). ```js /trigger.config.js export default defineConfig({ @@ -103,7 +103,7 @@ export default defineConfig({ ``` - [Build extensions](/config/config-file#extensions) allow you to hook into the build system and + [Build extensions](/config/extensions/overview) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx b/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx index d6fbfbc81d..d64f058520 100644 --- a/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx +++ b/docs/guides/frameworks/supabase-edge-functions-database-webhooks.mdx @@ -260,7 +260,7 @@ export default defineConfig({ ``` - [Build extensions](/config/config-file#extensions) allow you to hook into the build system and + [Build extensions](/config/extensions/overview) allow you to hook into the build system and customize the build process or the resulting bundle and container image (in the case of deploying). You can use pre-built extensions or create your own. diff --git a/docs/troubleshooting.mdx b/docs/troubleshooting.mdx index e9d71a5795..f67db8f14b 100644 --- a/docs/troubleshooting.mdx +++ b/docs/troubleshooting.mdx @@ -94,13 +94,14 @@ Your code is deployed separately from the rest of your app(s) so you need to mak ### `Error: @prisma/client did not initialize yet.` -Prisma uses code generation to create the client from your schema file. This means you need to add a bit of config so we can generate this file before your tasks run: [Read the guide](/config/config-file#prisma). +Prisma uses code generation to create the client from your schema file. This means you need to add a bit of config so we can generate this file before your tasks run: [Read the guide](/config/extensions/prismaExtension). ### `Parallel waits are not supported` In the current version, you can't perform more that one "wait" in parallel. Waits include: + - `wait.for()` - `wait.until()` - `task.triggerAndWait()` diff --git a/packages/build/package.json b/packages/build/package.json index 5ccdfd6305..e60903c869 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -23,6 +23,7 @@ "exports": { "./package.json": "./package.json", ".": "./src/index.ts", + "./internal": "./src/internal.ts", "./extensions": "./src/extensions/index.ts", "./extensions/core": "./src/extensions/core.ts", "./extensions/prisma": "./src/extensions/prisma.ts", @@ -36,6 +37,9 @@ }, "typesVersions": { "*": { + "internal": [ + "dist/commonjs/internal.d.ts" + ], "extensions": [ "dist/commonjs/extensions/index.d.ts" ], @@ -95,6 +99,17 @@ "default": "./dist/commonjs/index.js" } }, + "./internal": { + "import": { + "@triggerdotdev/source": "./src/internal.ts", + "types": "./dist/esm/internal.d.ts", + "default": "./dist/esm/internal.js" + }, + "require": { + "types": "./dist/commonjs/internal.d.ts", + "default": "./dist/commonjs/internal.js" + } + }, "./extensions": { "import": { "@triggerdotdev/source": "./src/extensions/index.ts", diff --git a/packages/build/src/extensions/core/additionalFiles.ts b/packages/build/src/extensions/core/additionalFiles.ts index 0a00a673ec..cc2a04e0e0 100644 --- a/packages/build/src/extensions/core/additionalFiles.ts +++ b/packages/build/src/extensions/core/additionalFiles.ts @@ -1,7 +1,5 @@ -import { relative, join, posix, dirname } from "node:path"; -import { glob } from "tinyglobby"; -import { copyFile, mkdir } from "node:fs/promises"; import { BuildExtension } from "@trigger.dev/core/v3/build"; +import { addAdditionalFilesToBuild } from "../../internal/additionalFiles.js"; export type AdditionalFilesOptions = { files: string[]; @@ -11,86 +9,7 @@ export function additionalFiles(options: AdditionalFilesOptions): BuildExtension return { name: "additionalFiles", async onBuildComplete(context, manifest) { - // Copy any static assets to the destination - const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, { - cwd: context.workingDir, - }); - - for (const { assets, matcher } of staticAssets) { - if (assets.length === 0) { - console.warn("No files found for matcher", matcher); - } - } - - await copyStaticAssets(staticAssets); + await addAdditionalFilesToBuild("additionalFiles", options, context, manifest); }, }; } - -type MatchedStaticAssets = { source: string; destination: string }[]; - -type FoundStaticAssetFiles = Array<{ - matcher: string; - assets: MatchedStaticAssets; -}>; - -async function findStaticAssetFiles( - matchers: string[], - destinationPath: string, - options?: { cwd?: string; ignore?: string[] } -): Promise { - const result: FoundStaticAssetFiles = []; - - for (const matcher of matchers) { - const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options); - - result.push({ matcher, assets }); - } - - return result; -} - -async function findStaticAssetsForMatcher( - matcher: string, - destinationPath: string, - options?: { cwd?: string; ignore?: string[] } -): Promise { - const result: MatchedStaticAssets = []; - - const files = await glob({ - patterns: [matcher], - cwd: options?.cwd, - ignore: options?.ignore ?? [], - onlyFiles: true, - absolute: true, - }); - - let matches = 0; - - for (const file of files) { - matches++; - - const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file) - .split(posix.sep) - .filter((p) => p !== "..") - .join(posix.sep); - - const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir); - - result.push({ - source: file, - destination: relativeDestinationPath, - }); - } - - return result; -} - -async function copyStaticAssets(staticAssetFiles: FoundStaticAssetFiles): Promise { - for (const { assets } of staticAssetFiles) { - for (const { source, destination } of assets) { - await mkdir(dirname(destination), { recursive: true }); - await copyFile(source, destination); - } - } -} diff --git a/packages/build/src/internal.ts b/packages/build/src/internal.ts new file mode 100644 index 0000000000..54f785a610 --- /dev/null +++ b/packages/build/src/internal.ts @@ -0,0 +1 @@ +export * from "./internal/additionalFiles.js"; diff --git a/packages/build/src/internal/additionalFiles.ts b/packages/build/src/internal/additionalFiles.ts new file mode 100644 index 0000000000..a815b53c9a --- /dev/null +++ b/packages/build/src/internal/additionalFiles.ts @@ -0,0 +1,106 @@ +import { BuildManifest } from "@trigger.dev/core/v3"; +import { BuildContext } from "@trigger.dev/core/v3/build"; +import { copyFile, mkdir } from "node:fs/promises"; +import { dirname, join, posix, relative } from "node:path"; +import { glob } from "tinyglobby"; + +export type AdditionalFilesOptions = { + files: string[]; +}; + +export async function addAdditionalFilesToBuild( + source: string, + options: AdditionalFilesOptions, + context: BuildContext, + manifest: BuildManifest +) { + // Copy any static assets to the destination + const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, { + cwd: context.workingDir, + }); + + for (const { assets, matcher } of staticAssets) { + if (assets.length === 0) { + context.logger.warn(`[${source}] No files found for matcher`, matcher); + } else { + context.logger.debug(`[${source}] Found ${assets.length} files for matcher`, matcher); + } + } + + await copyStaticAssets(staticAssets, source, context); +} + +type MatchedStaticAssets = { source: string; destination: string }[]; + +type FoundStaticAssetFiles = Array<{ + matcher: string; + assets: MatchedStaticAssets; +}>; + +async function findStaticAssetFiles( + matchers: string[], + destinationPath: string, + options?: { cwd?: string; ignore?: string[] } +): Promise { + const result: FoundStaticAssetFiles = []; + + for (const matcher of matchers) { + const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options); + + result.push({ matcher, assets }); + } + + return result; +} + +async function findStaticAssetsForMatcher( + matcher: string, + destinationPath: string, + options?: { cwd?: string; ignore?: string[] } +): Promise { + const result: MatchedStaticAssets = []; + + const files = await glob({ + patterns: [matcher], + cwd: options?.cwd, + ignore: options?.ignore ?? [], + onlyFiles: true, + absolute: true, + }); + + let matches = 0; + + for (const file of files) { + matches++; + + const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file) + .split(posix.sep) + .filter((p) => p !== "..") + .join(posix.sep); + + const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir); + + result.push({ + source: file, + destination: relativeDestinationPath, + }); + } + + return result; +} + +async function copyStaticAssets( + staticAssetFiles: FoundStaticAssetFiles, + sourceName: string, + context: BuildContext +): Promise { + for (const { assets } of staticAssetFiles) { + for (const { source, destination } of assets) { + await mkdir(dirname(destination), { recursive: true }); + + context.logger.debug(`[${sourceName}] Copying ${source} to ${destination}`); + + await copyFile(source, destination); + } + } +} diff --git a/packages/core/src/v3/apiClient/index.ts b/packages/core/src/v3/apiClient/index.ts index d57899a311..1ba853689f 100644 --- a/packages/core/src/v3/apiClient/index.ts +++ b/packages/core/src/v3/apiClient/index.ts @@ -61,14 +61,13 @@ import { SubscribeToRunsQueryParams, UpdateEnvironmentVariableParams, } from "./types.js"; -import type { AsyncIterableStream } from "./stream.js"; +import { AsyncIterableStream } from "../streams/asyncIterableStream.js"; export type { CreateEnvironmentVariableParams, ImportEnvironmentVariablesParams, SubscribeToRunsQueryParams, UpdateEnvironmentVariableParams, - AsyncIterableStream, }; export type ClientTriggerOptions = { diff --git a/packages/core/src/v3/apiClient/runStream.ts b/packages/core/src/v3/apiClient/runStream.ts index bd73937442..92d74929be 100644 --- a/packages/core/src/v3/apiClient/runStream.ts +++ b/packages/core/src/v3/apiClient/runStream.ts @@ -16,12 +16,11 @@ import { } from "../utils/ioSerialization.js"; import { ApiError } from "./errors.js"; import { ApiClient } from "./index.js"; +import { LineTransformStream, zodShapeStream } from "./stream.js"; import { AsyncIterableStream, createAsyncIterableReadable, - LineTransformStream, - zodShapeStream, -} from "./stream.js"; +} from "../streams/asyncIterableStream.js"; export type RunShape = TRunTypes extends AnyRunTypes ? { diff --git a/packages/core/src/v3/apiClient/stream.ts b/packages/core/src/v3/apiClient/stream.ts index fd027ab665..77487954cf 100644 --- a/packages/core/src/v3/apiClient/stream.ts +++ b/packages/core/src/v3/apiClient/stream.ts @@ -9,6 +9,7 @@ import { type Row, type ShapeStreamInterface, } from "@electric-sql/client"; +import { AsyncIterableStream, createAsyncIterableStream } from "../streams/asyncIterableStream.js"; export type ZodShapeStreamOptions = { headers?: Record; @@ -82,57 +83,6 @@ export function zodShapeStream( }; } -export type AsyncIterableStream = AsyncIterable & ReadableStream; - -export function createAsyncIterableStream( - source: ReadableStream, - transformer: Transformer -): AsyncIterableStream { - const transformedStream: any = source.pipeThrough(new TransformStream(transformer)); - - transformedStream[Symbol.asyncIterator] = () => { - const reader = transformedStream.getReader(); - return { - async next(): Promise> { - const { done, value } = await reader.read(); - return done ? { done: true, value: undefined } : { done: false, value }; - }, - }; - }; - - return transformedStream; -} - -export function createAsyncIterableReadable( - source: ReadableStream, - transformer: Transformer, - signal: AbortSignal -): AsyncIterableStream { - return new ReadableStream({ - async start(controller) { - const transformedStream = source.pipeThrough(new TransformStream(transformer)); - const reader = transformedStream.getReader(); - - signal.addEventListener("abort", () => { - queueMicrotask(() => { - reader.cancel(); - controller.close(); - }); - }); - - while (true) { - const { done, value } = await reader.read(); - if (done) { - controller.close(); - break; - } - - controller.enqueue(value); - } - }, - }) as AsyncIterableStream; -} - class ReadableShapeStream = Row> { readonly #stream: ShapeStreamInterface; readonly #currentState: Map = new Map(); diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index ee29162924..5638b98768 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -22,6 +22,7 @@ export * from "./types/index.js"; export { links } from "./links.js"; export * from "./jwt.js"; export * from "./idempotencyKeys.js"; +export * from "./streams/asyncIterableStream.js"; export * from "./utils/getEnv.js"; export { formatDuration, diff --git a/packages/core/src/v3/logger/index.ts b/packages/core/src/v3/logger/index.ts index 9c50b06610..b569a2869f 100644 --- a/packages/core/src/v3/logger/index.ts +++ b/packages/core/src/v3/logger/index.ts @@ -1,6 +1,6 @@ import { NoopTaskLogger, TaskLogger } from "./taskLogger.js"; import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js"; -import { Span } from "@opentelemetry/api"; +import { Span, SpanOptions } from "@opentelemetry/api"; const API_NAME = "logger"; @@ -47,8 +47,12 @@ export class LoggerAPI implements TaskLogger { this.#getTaskLogger().error(message, metadata); } - public trace(name: string, fn: (span: Span) => Promise): Promise { - return this.#getTaskLogger().trace(name, fn); + public trace(name: string, fn: (span: Span) => Promise, options?: SpanOptions): Promise { + return this.#getTaskLogger().trace(name, fn, options); + } + + public startSpan(name: string, options?: SpanOptions): Span { + return this.#getTaskLogger().startSpan(name, options); } #getTaskLogger(): TaskLogger { diff --git a/packages/core/src/v3/logger/taskLogger.ts b/packages/core/src/v3/logger/taskLogger.ts index e194f28a54..48d7968cad 100644 --- a/packages/core/src/v3/logger/taskLogger.ts +++ b/packages/core/src/v3/logger/taskLogger.ts @@ -24,6 +24,7 @@ export interface TaskLogger { warn(message: string, properties?: Record): void; error(message: string, properties?: Record): void; trace(name: string, fn: (span: Span) => Promise, options?: SpanOptions): Promise; + startSpan(name: string, options?: SpanOptions): Span; } export class OtelTaskLogger implements TaskLogger { @@ -90,6 +91,10 @@ export class OtelTaskLogger implements TaskLogger { return this._config.tracer.startActiveSpan(name, fn, options); } + startSpan(name: string, options?: SpanOptions): Span { + return this._config.tracer.startSpan(name, options); + } + #getTimestampInHrTime(): ClockTime { return clock.preciseNow(); } @@ -104,6 +109,9 @@ export class NoopTaskLogger implements TaskLogger { trace(name: string, fn: (span: Span) => Promise): Promise { return fn({} as Span); } + startSpan(): Span { + return {} as Span; + } } function safeJsonProcess(value?: Record): Record | undefined { diff --git a/packages/core/src/v3/runMetadata/index.ts b/packages/core/src/v3/runMetadata/index.ts index e39213cd62..edc8475fb8 100644 --- a/packages/core/src/v3/runMetadata/index.ts +++ b/packages/core/src/v3/runMetadata/index.ts @@ -1,5 +1,5 @@ import { DeserializedJson } from "../../schemas/json.js"; -import { AsyncIterableStream } from "../apiClient/stream.js"; +import { AsyncIterableStream } from "../streams/asyncIterableStream.js"; import { getGlobal, registerGlobal } from "../utils/globals.js"; import { ApiRequestOptions } from "../zodfetch.js"; import { NoopRunMetadataManager } from "./noopManager.js"; diff --git a/packages/core/src/v3/runMetadata/manager.ts b/packages/core/src/v3/runMetadata/manager.ts index f847d644c4..74c0203b0a 100644 --- a/packages/core/src/v3/runMetadata/manager.ts +++ b/packages/core/src/v3/runMetadata/manager.ts @@ -1,12 +1,12 @@ import { dequal } from "dequal/lite"; import { DeserializedJson } from "../../schemas/json.js"; import { ApiClient } from "../apiClient/index.js"; -import { AsyncIterableStream } from "../apiClient/stream.js"; import { FlushedRunMetadata, RunMetadataChangeOperation } from "../schemas/common.js"; import { ApiRequestOptions } from "../zodfetch.js"; import { MetadataStream } from "./metadataStream.js"; import { applyMetadataOperations } from "./operations.js"; import { RunMetadataManager, RunMetadataUpdater } from "./types.js"; +import { AsyncIterableStream } from "../streams/asyncIterableStream.js"; const MAXIMUM_ACTIVE_STREAMS = 5; const MAXIMUM_TOTAL_STREAMS = 10; diff --git a/packages/core/src/v3/runMetadata/noopManager.ts b/packages/core/src/v3/runMetadata/noopManager.ts index 03758d9032..85d1596c36 100644 --- a/packages/core/src/v3/runMetadata/noopManager.ts +++ b/packages/core/src/v3/runMetadata/noopManager.ts @@ -1,5 +1,5 @@ import { DeserializedJson } from "../../schemas/json.js"; -import { AsyncIterableStream } from "../apiClient/stream.js"; +import { AsyncIterableStream } from "../streams/asyncIterableStream.js"; import { ApiRequestOptions } from "../zodfetch.js"; import type { RunMetadataManager, RunMetadataUpdater } from "./types.js"; diff --git a/packages/core/src/v3/runMetadata/types.ts b/packages/core/src/v3/runMetadata/types.ts index 10cb506ec1..53a3a21133 100644 --- a/packages/core/src/v3/runMetadata/types.ts +++ b/packages/core/src/v3/runMetadata/types.ts @@ -1,5 +1,5 @@ import { DeserializedJson } from "../../schemas/json.js"; -import { AsyncIterableStream } from "../apiClient/stream.js"; +import { AsyncIterableStream } from "../streams/asyncIterableStream.js"; import { ApiRequestOptions } from "../zodfetch.js"; export interface RunMetadataUpdater { diff --git a/packages/core/src/v3/streams/asyncIterableStream.ts b/packages/core/src/v3/streams/asyncIterableStream.ts new file mode 100644 index 0000000000..6c9ad1ea12 --- /dev/null +++ b/packages/core/src/v3/streams/asyncIterableStream.ts @@ -0,0 +1,97 @@ +export type AsyncIterableStream = AsyncIterable & ReadableStream; + +export function createAsyncIterableStream( + source: ReadableStream, + transformer: Transformer +): AsyncIterableStream { + const transformedStream: any = source.pipeThrough(new TransformStream(transformer)); + + transformedStream[Symbol.asyncIterator] = () => { + const reader = transformedStream.getReader(); + return { + async next(): Promise> { + const { done, value } = await reader.read(); + return done ? { done: true, value: undefined } : { done: false, value }; + }, + }; + }; + + return transformedStream; +} + +export function createAsyncIterableReadable( + source: ReadableStream, + transformer: Transformer, + signal: AbortSignal +): AsyncIterableStream { + return new ReadableStream({ + async start(controller) { + const transformedStream = source.pipeThrough(new TransformStream(transformer)); + const reader = transformedStream.getReader(); + + signal.addEventListener("abort", () => { + queueMicrotask(() => { + reader.cancel(); + controller.close(); + }); + }); + + while (true) { + const { done, value } = await reader.read(); + if (done) { + controller.close(); + break; + } + + controller.enqueue(value); + } + }, + }) as AsyncIterableStream; +} + +export function createAsyncIterableStreamFromAsyncIterable( + asyncIterable: AsyncIterable, + transformer: Transformer, + signal?: AbortSignal +): AsyncIterableStream { + const stream = new ReadableStream({ + async start(controller) { + try { + if (signal) { + signal.addEventListener("abort", () => { + controller.close(); + }); + } + + const iterator = asyncIterable[Symbol.asyncIterator](); + + while (true) { + if (signal?.aborted) { + break; + } + + const { done, value } = await iterator.next(); + + if (done) { + controller.close(); + break; + } + + controller.enqueue(value); + } + } catch (error) { + controller.error(error); + } + }, + cancel() { + // If the stream is a tinyexec process with a kill method, kill it + if ("kill" in asyncIterable) { + (asyncIterable as any).kill(); + } + }, + }); + + const transformedStream = stream.pipeThrough(new TransformStream(transformer)); + + return transformedStream as AsyncIterableStream; +} diff --git a/packages/python/README.md b/packages/python/README.md index 7dd697b46c..805310f2ae 100644 --- a/packages/python/README.md +++ b/packages/python/README.md @@ -14,7 +14,7 @@ This extension introduces the pythonExtension build extension, whic - run: Executes Python commands with proper environment setup. - runInline: Executes inline Python code directly from Node. - runScript: Executes standalone .py script files. -- **Custom Python Path:** In development, you can configure pythonBinaryPath to point to a custom Python installation. +- **Custom Python Path:** In development, you can configure devPythonBinaryPath to point to a custom Python installation. ## Usage @@ -22,7 +22,7 @@ This extension introduces the pythonExtension build extension, whic ```typescript import { defineConfig } from "@trigger.dev/sdk/v3"; -import pythonExtension from "@trigger.dev/python/extension"; +import { pythonExtension } from "@trigger.dev/python/extension"; export default defineConfig({ project: "", @@ -30,8 +30,8 @@ export default defineConfig({ extensions: [ pythonExtension({ requirementsFile: "./requirements.txt", // Optional: Path to your requirements file - pythonBinaryPath: path.join(rootDir, `.venv/bin/python`), // Optional: Custom Python binary path - scripts: ["my_script.py"], // List of Python scripts to include + devPythonBinaryPath: ".venv/bin/python", // Optional: Custom Python binary path + scripts: ["src/python/**/*.py"], // Glob pattern for Python scripts }), ], }, @@ -40,13 +40,34 @@ export default defineConfig({ 2. (Optional) Create a requirements.txt file in your project root with the necessary Python dependencies. +```plaintext title="requirements.txt" +pandas==1.3.3 +numpy==1.21.2 +``` + +```typescript title="trigger.config.ts" +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { pythonExtension } from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + pythonExtension({ + requirementsFile: "./requirements.txt", + }), + ], + }, +}); +``` + 3. Execute Python scripts within your tasks using one of the provided functions: ### Running a Python Script ```typescript import { task } from "@trigger.dev/sdk/v3"; -import python from "@trigger.dev/python"; +import { python } from "@trigger.dev/python"; export const myScript = task({ id: "my-python-script", @@ -55,13 +76,29 @@ export const myScript = task({ return result.stdout; }, }); + +export const myStreamingScript = task({ + id: "my-streaming-python-script", + run: async () => { + // You can also stream the output of the script + const result = python.stream.runScript("my_script.py", ["hello", "world"]); + + // result is an async iterable/readable stream + for await (const chunk of streamingResult) { + logger.debug("convert-url-to-markdown", { + url: payload.url, + chunk, + }); + } + }, +}); ``` ### Running Inline Python Code ```typescript import { task } from "@trigger.dev/sdk/v3"; -import python from "@trigger.dev/python"; +import { python } from "@trigger.dev/python"; export const myTask = task({ id: "to_datetime-task", @@ -69,7 +106,7 @@ export const myTask = task({ const result = await python.runInline(` import pandas as pd -pandas.to_datetime("${+new Date() / 1000}") +pd.to_datetime("${+new Date() / 1000}") `); return result.stdout; }, @@ -80,7 +117,7 @@ pandas.to_datetime("${+new Date() / 1000}") ```typescript import { task } from "@trigger.dev/sdk/v3"; -import python from "@trigger.dev/python"; +import { python } from "@trigger.dev/python"; export const pythonVersionTask = task({ id: "python-version-task", @@ -94,7 +131,6 @@ export const pythonVersionTask = task({ ## Limitations - This is a **partial implementation** and does not provide full Python support as an execution runtime for tasks. -- Only basic Python script execution is supported; scripts are not automatically copied to staging/production containers. - Manual intervention may be required for installing and configuring binary dependencies in development environments. ## Additional Information diff --git a/packages/python/package.json b/packages/python/package.json index 7d032b6c8a..c7011797a2 100644 --- a/packages/python/package.json +++ b/packages/python/package.json @@ -45,9 +45,7 @@ "check-exports": "attw --pack ." }, "dependencies": { - "@trigger.dev/build": "workspace:3.3.16", "@trigger.dev/core": "workspace:3.3.16", - "@trigger.dev/sdk": "workspace:3.3.16", "tinyexec": "^0.3.2" }, "devDependencies": { @@ -57,7 +55,13 @@ "typescript": "^5.5.4", "tsx": "4.17.0", "esbuild": "^0.23.0", - "@arethetypeswrong/cli": "^0.15.4" + "@arethetypeswrong/cli": "^0.15.4", + "@trigger.dev/build": "workspace:3.3.16", + "@trigger.dev/sdk": "workspace:3.3.16" + }, + "peerDependencies": { + "@trigger.dev/sdk": "workspace:^3.3.16", + "@trigger.dev/build": "workspace:^3.3.16" }, "engines": { "node": ">=18.20.0" diff --git a/packages/python/src/extension.ts b/packages/python/src/extension.ts index ea0c4140fb..53b5c2c8cf 100644 --- a/packages/python/src/extension.ts +++ b/packages/python/src/extension.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; import assert from "node:assert"; -import { additionalFiles } from "@trigger.dev/build/extensions/core"; +import { addAdditionalFilesToBuild } from "@trigger.dev/build/internal"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; @@ -16,7 +16,7 @@ export type PythonOptions = { * * Example: `/usr/bin/python3` or `C:\\Python39\\python.exe` */ - pythonBinaryPath?: string; + devPythonBinaryPath?: string; /** * An array of glob patterns that specify which Python scripts are allowed to be executed. * @@ -57,13 +57,18 @@ class PythonExtension implements BuildExtension { } async onBuildComplete(context: BuildContext, manifest: BuildManifest) { - await additionalFiles({ - files: this.options.scripts ?? [], - }).onBuildComplete!(context, manifest); + await addAdditionalFilesToBuild( + "pythonExtension", + { + files: this.options.scripts ?? [], + }, + context, + manifest + ); if (context.target === "dev") { - if (this.options.pythonBinaryPath) { - process.env.PYTHON_BIN_PATH = this.options.pythonBinaryPath; + if (this.options.devPythonBinaryPath) { + process.env.PYTHON_BIN_PATH = this.options.devPythonBinaryPath; } return; @@ -93,27 +98,59 @@ class PythonExtension implements BuildExtension { }, }); - context.addLayer({ - id: "python-dependencies", - build: { - env: { - REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "", + if (this.options.requirementsFile) { + if (this.options.requirements) { + context.logger.warn( + `[pythonExtension] Both options.requirements and options.requirementsFile are specified. requirements will be ignored.` + ); + } + + // Copy requirements file to the container + await addAdditionalFilesToBuild( + "pythonExtension", + { + files: [this.options.requirementsFile], }, - }, - image: { - instructions: splitAndCleanComments(` + context, + manifest + ); + + // Add a layer to the build that installs the requirements + context.addLayer({ + id: "python-dependencies", + image: { + instructions: splitAndCleanComments(` + # Copy the requirements file + COPY ${this.options.requirementsFile} . + # Install dependencies + RUN pip install --no-cache-dir -r ${this.options.requirementsFile} + `), + }, + deploy: { + override: true, + }, + }); + } else if (this.options.requirements) { + context.addLayer({ + id: "python-dependencies", + build: { + env: { + REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "", + }, + }, + image: { + instructions: splitAndCleanComments(` ARG REQUIREMENTS_CONTENT RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt # Install dependencies RUN pip install --no-cache-dir -r requirements.txt `), - }, - deploy: { - override: true, - }, - }); + }, + deploy: { + override: true, + }, + }); + } } } - -export default pythonExtension; diff --git a/packages/python/src/index.ts b/packages/python/src/index.ts index aefcfddbe4..f152cc01e5 100644 --- a/packages/python/src/index.ts +++ b/packages/python/src/index.ts @@ -1,63 +1,277 @@ -import fs from "node:fs"; -import assert from "node:assert"; +import { + AsyncIterableStream, + createAsyncIterableStreamFromAsyncIterable, + SemanticInternalAttributes, +} from "@trigger.dev/core/v3"; import { logger } from "@trigger.dev/sdk/v3"; -import { x, Options as XOptions, Result } from "tinyexec"; +import assert from "node:assert"; +import fs from "node:fs"; +import { Result, x, Options as XOptions } from "tinyexec"; +import { createTempFileSync, withTempFile } from "./utils/tempFiles.js"; -export const run = async ( - scriptArgs: string[] = [], - options: Partial = {} -): Promise => { - const pythonBin = process.env.PYTHON_BIN_PATH || "python"; +export type PythonExecOptions = Partial & { + env?: { [key: string]: string | undefined }; +}; - return await logger.trace("Python call", async (span) => { - span.addEvent("Properties", { - command: `${pythonBin} ${scriptArgs.join(" ")}`, - }); +export const python = { + async run(scriptArgs: string[] = [], options: PythonExecOptions = {}): Promise { + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; - const result = await x(pythonBin, scriptArgs, { - ...options, - throwOnError: false, // Ensure errors are handled manually - }); + return await logger.trace( + "python.run()", + async (span) => { + const result = await x(pythonBin, scriptArgs, { + ...options, + nodeOptions: { + ...(options.nodeOptions || {}), + env: { + ...process.env, + ...options.env, + }, + }, + throwOnError: false, // Ensure errors are handled manually + }); - span.addEvent("Output", { ...result }); + if (result.exitCode) { + span.setAttribute("exitCode", result.exitCode); + } - if (result.exitCode !== 0) { - logger.error(result.stderr, { ...result }); - throw new Error(`Python command exited with non-zero code ${result.exitCode}`); - } + if (result.exitCode !== 0) { + throw new Error( + `${scriptArgs.join(" ")} exited with a non-zero code ${result.exitCode}:\n${ + result.stderr + }` + ); + } - return result; - }); -}; + return result; + }, + { + attributes: { + pythonBin, + args: scriptArgs.join(" "), + [SemanticInternalAttributes.STYLE_ICON]: "brand-python", + }, + } + ); + }, -export const runScript = ( - scriptPath: string, - scriptArgs: string[] = [], - options: Partial = {} -) => { - assert(scriptPath, "Script path is required"); - assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); + async runScript( + scriptPath: string, + scriptArgs: string[] = [], + options: PythonExecOptions = {} + ): Promise { + assert(scriptPath, "Script path is required"); + assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); - return run([scriptPath, ...scriptArgs], options); -}; + return await logger.trace( + "python.runScript()", + async (span) => { + span.setAttribute("scriptPath", scriptPath); + + const result = await x( + process.env.PYTHON_BIN_PATH || "python", + [scriptPath, ...scriptArgs], + { + ...options, + nodeOptions: { + ...(options.nodeOptions || {}), + env: { + ...process.env, + ...options.env, + }, + }, + throwOnError: false, + } + ); + + if (result.exitCode) { + span.setAttribute("exitCode", result.exitCode); + } -export const runInline = async (scriptContent: string, options: Partial = {}) => { - assert(scriptContent, "Script content is required"); + if (result.exitCode !== 0) { + throw new Error( + `${scriptPath} ${scriptArgs.join(" ")} exited with a non-zero code ${ + result.exitCode + }:\n${result.stderr}` + ); + } - const tmpFile = `/tmp/script_${Date.now()}.py`; - await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); + return result; + }, + { + attributes: { + pythonBin: process.env.PYTHON_BIN_PATH || "python", + scriptPath, + args: scriptArgs.join(" "), + [SemanticInternalAttributes.STYLE_ICON]: "brand-python", + }, + } + ); + }, - try { - return await runScript(tmpFile, [], options); - } finally { - try { - await fs.promises.unlink(tmpFile); - } catch (error) { - logger.warn(`Failed to clean up temporary file ${tmpFile}:`, { - error: (error as Error).stack || (error as Error).message, + async runInline(scriptContent: string, options: PythonExecOptions = {}): Promise { + assert(scriptContent, "Script content is required"); + + return await logger.trace( + "python.runInline()", + async (span) => { + span.setAttribute("contentLength", scriptContent.length); + + // Using the withTempFile utility to handle the temporary file + return await withTempFile( + `script_${Date.now()}.py`, + async (tempFilePath) => { + span.setAttribute("tempFilePath", tempFilePath); + + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; + const result = await x(pythonBin, [tempFilePath], { + ...options, + nodeOptions: { + ...(options.nodeOptions || {}), + env: { + ...process.env, + ...options.env, + }, + }, + throwOnError: false, + }); + + if (result.exitCode) { + span.setAttribute("exitCode", result.exitCode); + } + + if (result.exitCode !== 0) { + throw new Error( + `Inline script exited with a non-zero code ${result.exitCode}:\n${result.stderr}` + ); + } + + return result; + }, + scriptContent + ); + }, + { + attributes: { + pythonBin: process.env.PYTHON_BIN_PATH || "python", + contentPreview: + scriptContent.substring(0, 100) + (scriptContent.length > 100 ? "..." : ""), + [SemanticInternalAttributes.STYLE_ICON]: "brand-python", + }, + } + ); + }, + // Stream namespace for streaming functions + stream: { + run(scriptArgs: string[] = [], options: PythonExecOptions = {}): AsyncIterableStream { + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; + + const pythonProcess = x(pythonBin, scriptArgs, { + ...options, + nodeOptions: { + ...(options.nodeOptions || {}), + env: { + ...process.env, + ...options.env, + }, + }, + throwOnError: false, + }); + + const span = logger.startSpan("python.stream.run()", { + attributes: { + pythonBin, + args: scriptArgs.join(" "), + [SemanticInternalAttributes.STYLE_ICON]: "brand-python", + }, + }); + + return createAsyncIterableStreamFromAsyncIterable(pythonProcess, { + transform: (chunk, controller) => { + controller.enqueue(chunk); + }, + flush: () => { + span.end(); + }, + }); + }, + runScript( + scriptPath: string, + scriptArgs: string[] = [], + options: PythonExecOptions = {} + ): AsyncIterableStream { + assert(scriptPath, "Script path is required"); + assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); + + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; + + const pythonProcess = x(pythonBin, [scriptPath, ...scriptArgs], { + ...options, + nodeOptions: { + ...(options.nodeOptions || {}), + env: { + ...process.env, + ...options.env, + }, + }, + throwOnError: false, + }); + + const span = logger.startSpan("python.stream.runScript()", { + attributes: { + pythonBin, + scriptPath, + args: scriptArgs.join(" "), + [SemanticInternalAttributes.STYLE_ICON]: "brand-python", + }, + }); + + return createAsyncIterableStreamFromAsyncIterable(pythonProcess, { + transform: (chunk, controller) => { + controller.enqueue(chunk); + }, + flush: () => { + span.end(); + }, + }); + }, + runInline(scriptContent: string, options: PythonExecOptions = {}): AsyncIterableStream { + assert(scriptContent, "Script content is required"); + + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; + + const pythonScriptPath = createTempFileSync(`script_${Date.now()}.py`, scriptContent); + + const pythonProcess = x(pythonBin, [pythonScriptPath], { + ...options, + nodeOptions: { + ...(options.nodeOptions || {}), + env: { + ...process.env, + ...options.env, + }, + }, + throwOnError: false, }); - } - } -}; -export default { run, runScript, runInline }; + const span = logger.startSpan("python.stream.runInline()", { + attributes: { + pythonBin, + contentPreview: + scriptContent.substring(0, 100) + (scriptContent.length > 100 ? "..." : ""), + [SemanticInternalAttributes.STYLE_ICON]: "brand-python", + }, + }); + + return createAsyncIterableStreamFromAsyncIterable(pythonProcess, { + transform: (chunk, controller) => { + controller.enqueue(chunk); + }, + flush: () => { + span.end(); + }, + }); + }, + }, +}; diff --git a/packages/python/src/utils/tempFiles.ts b/packages/python/src/utils/tempFiles.ts new file mode 100644 index 0000000000..819e486463 --- /dev/null +++ b/packages/python/src/utils/tempFiles.ts @@ -0,0 +1,39 @@ +import { mkdtempSync, writeFileSync } from "node:fs"; +import { mkdtemp, writeFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +/** + * Creates a temporary file with a custom filename, passes it to the callback function, and ensures cleanup + * @param filename The filename to use for the temporary file + * @param callback Function that receives the path to the temporary file + * @param content Optional content to write to the file + * @returns Whatever the callback returns + */ +export async function withTempFile( + filename: string, + callback: (filePath: string) => Promise, + content: string | Buffer = "" +): Promise { + // Create temporary directory with random suffix + const tempDir = await mkdtemp(join(tmpdir(), "app-")); + const tempFile = join(tempDir, filename); + + try { + // Write to the temporary file with appropriate permissions + await writeFile(tempFile, content, { mode: 0o600 }); + // Use the file + return await callback(tempFile); + } finally { + // Clean up + await rm(tempDir, { recursive: true, force: true }); + } +} + +export function createTempFileSync(filename: string, content: string | Buffer = ""): string { + const tempDir = mkdtempSync(join(tmpdir(), "app-")); + const tempFile = join(tempDir, filename); + + writeFileSync(tempFile, content, { mode: 0o600 }); + return tempFile; +} diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b3fb9ce373..df08e6c8eb 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1419,15 +1419,9 @@ importers: packages/python: dependencies: - '@trigger.dev/build': - specifier: workspace:3.3.16 - version: link:../build '@trigger.dev/core': specifier: workspace:3.3.16 version: link:../core - '@trigger.dev/sdk': - specifier: workspace:3.3.16 - version: link:../trigger-sdk tinyexec: specifier: ^0.3.2 version: 0.3.2 @@ -1435,6 +1429,12 @@ importers: '@arethetypeswrong/cli': specifier: ^0.15.4 version: 0.15.4 + '@trigger.dev/build': + specifier: workspace:3.3.16 + version: link:../build + '@trigger.dev/sdk': + specifier: workspace:3.3.16 + version: link:../trigger-sdk '@types/node': specifier: 20.14.14 version: 20.14.14 @@ -1781,6 +1781,28 @@ importers: specifier: ^5 version: 5.5.4 + references/python-catalog: + dependencies: + '@trigger.dev/python': + specifier: workspace:* + version: link:../../packages/python + '@trigger.dev/sdk': + specifier: workspace:* + version: link:../../packages/trigger-sdk + zod: + specifier: 3.23.8 + version: 3.23.8 + devDependencies: + '@trigger.dev/build': + specifier: workspace:* + version: link:../../packages/build + trigger.dev: + specifier: workspace:* + version: link:../../packages/cli-v3 + typescript: + specifier: ^5.5.4 + version: 5.5.4 + references/test-tasks: dependencies: '@trigger.dev/sdk': diff --git a/references/python-catalog/package.json b/references/python-catalog/package.json new file mode 100644 index 0000000000..171a1e3ea4 --- /dev/null +++ b/references/python-catalog/package.json @@ -0,0 +1,20 @@ +{ + "name": "references-python-catalog", + "private": true, + "type": "module", + "scripts": { + "dev": "trigger dev", + "deploy": "trigger deploy --self-hosted --load-image", + "install-python-deps": "uv pip sync requirements.txt" + }, + "dependencies": { + "@trigger.dev/sdk": "workspace:*", + "@trigger.dev/python": "workspace:*", + "zod": "3.23.8" + }, + "devDependencies": { + "@trigger.dev/build": "workspace:*", + "trigger.dev": "workspace:*", + "typescript": "^5.5.4" + } +} \ No newline at end of file diff --git a/references/python-catalog/requirements.txt b/references/python-catalog/requirements.txt new file mode 100644 index 0000000000..412b45058b --- /dev/null +++ b/references/python-catalog/requirements.txt @@ -0,0 +1,5 @@ +html2text==2024.2.26 +requests==2.32.3 +urllib3==2.3.0 +idna==3.10 +certifi==2025.1.31 \ No newline at end of file diff --git a/references/python-catalog/src/python/html2text_url.py b/references/python-catalog/src/python/html2text_url.py new file mode 100644 index 0000000000..df56672e06 --- /dev/null +++ b/references/python-catalog/src/python/html2text_url.py @@ -0,0 +1,37 @@ +import html2text +import requests +import argparse +import sys + +def fetch_html(url): + """Fetch HTML content from a URL.""" + try: + response = requests.get(url) + response.raise_for_status() # Raise an exception for HTTP errors + return response.text + except requests.exceptions.RequestException as e: + print(f"Error fetching URL: {e}", file=sys.stderr) + sys.exit(1) + +def main(): + # Set up command line argument parsing + parser = argparse.ArgumentParser(description='Convert HTML from a URL to plain text.') + parser.add_argument('url', help='The URL to fetch HTML from') + parser.add_argument('--ignore-links', action='store_true', + help='Ignore converting links from HTML') + + args = parser.parse_args() + + # Fetch HTML from the URL + html_content = fetch_html(args.url) + + # Configure html2text + h = html2text.HTML2Text() + h.ignore_links = args.ignore_links + + # Convert HTML to text and print + text_content = h.handle(html_content) + print(text_content) + +if __name__ == "__main__": + main() diff --git a/references/python-catalog/src/trigger/pythonTasks.ts b/references/python-catalog/src/trigger/pythonTasks.ts new file mode 100644 index 0000000000..a358c4be5d --- /dev/null +++ b/references/python-catalog/src/trigger/pythonTasks.ts @@ -0,0 +1,68 @@ +import { logger, schemaTask, task } from "@trigger.dev/sdk/v3"; +import { python } from "@trigger.dev/python"; +import { z } from "zod"; + +export const convertUrlToMarkdown = schemaTask({ + id: "convert-url-to-markdown", + schema: z.object({ + url: z.string().url(), + }), + run: async (payload) => { + const result = await python.runScript("./src/python/html2text_url.py", [payload.url]); + + logger.debug("convert-url-to-markdown", { + url: payload.url, + output: result.stdout, + }); + + const streamingResult = python.stream.runScript("./src/python/html2text_url.py", [payload.url]); + + for await (const chunk of streamingResult) { + logger.debug("convert-url-to-markdown", { + url: payload.url, + chunk, + }); + } + }, +}); + +export const pythonRunInlineTask = task({ + id: "python-run-inline", + run: async () => { + const result = await python.runInline( + ` +import os +import html2text as h2t + +h = h2t.HTML2Text() + +print(h.handle("

Hello, world!")) +print(f"API Key: {os.environ['OPENAI_API_KEY']}") +`, + { + env: { + OPENAI_API_KEY: "sk-1234567890", + }, + } + ); + + console.log(result.stdout); + + const streamingResult = python.stream.runInline(` +import html2text as h2t + +h = h2t.HTML2Text() + +print(h.handle("

Hello, world!")) +print(h.handle("

Hello, world!")) +`); + + for await (const chunk of streamingResult) { + logger.debug("python-run-inline", { + chunk, + }); + } + + return result.stdout; + }, +}); diff --git a/references/python-catalog/trigger.config.ts b/references/python-catalog/trigger.config.ts new file mode 100644 index 0000000000..5d2e9e05f2 --- /dev/null +++ b/references/python-catalog/trigger.config.ts @@ -0,0 +1,29 @@ +import { defineConfig } from "@trigger.dev/sdk/v3"; +import { pythonExtension } from "@trigger.dev/python/extension"; + +export default defineConfig({ + runtime: "node", + project: "proj_hbsqkjxevkyuklehrgrp", + machine: "small-1x", + maxDuration: 3600, + dirs: ["./src/trigger"], + build: { + extensions: [ + pythonExtension({ + requirementsFile: "./requirements.txt", // Optional: Path to your requirements file + devPythonBinaryPath: `.venv/bin/python`, // Optional: Custom Python binary path + scripts: ["src/python/**/*.py"], // List of Python scripts to include + }), + ], + }, + retries: { + enabledInDev: true, + default: { + maxAttempts: 3, + minTimeoutInMs: 1_000, + maxTimeoutInMs: 5_000, + factor: 1.6, + randomize: true, + }, + }, +}); diff --git a/references/python-catalog/tsconfig.json b/references/python-catalog/tsconfig.json new file mode 100644 index 0000000000..635ecdb766 --- /dev/null +++ b/references/python-catalog/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compilerOptions": { + "target": "esnext", + "module": "NodeNext", + "moduleResolution": "NodeNext", + "esModuleInterop": true, + "strict": true, + "outDir": "dist", + "skipLibCheck": true, + "customConditions": ["@triggerdotdev/source"], + "jsx": "preserve", + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "lib": ["DOM", "DOM.Iterable"], + "paths": { + "@/*": ["./src/*"] + } + }, + "include": ["./src/**/*.ts", "trigger.config.ts"] +}