From 2a4aa8c7fffe1874f661613c9727006b0e309fb5 Mon Sep 17 00:00:00 2001 From: Victor Duarte Date: Sun, 9 Feb 2025 14:48:16 -0300 Subject: [PATCH 01/29] add PythonExtension --- packages/build/src/extensions/python.ts | 116 ++++++++++++++++++++++++ 1 file changed, 116 insertions(+) create mode 100644 packages/build/src/extensions/python.ts diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts new file mode 100644 index 0000000000..1faef7ca7f --- /dev/null +++ b/packages/build/src/extensions/python.ts @@ -0,0 +1,116 @@ +import fs from "node:fs"; +import { $ } from "execa"; +import { assert } from "@std/assert"; +import { BuildManifest } from "@trigger.dev/core/v3"; +import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; +import { logger } from "@trigger.dev/sdk/v3"; + +export type PythonOptions = { + requirements?: string[]; + requirementsFile?: string; + /** + * [Dev-only] The path to the python binary. + * + * @remarks + * This option is typically used during local development or in specific testing environments + * where a particular Python installation needs to be targeted. It should point to the full path of the python executable. + * + * Example: `/usr/bin/python3` or `C:\\Python39\\python.exe` + */ + pythonBinaryPath?: string; +}; + +export function pythonExtension(options: PythonOptions = {}): BuildExtension { + return new PythonExtension(options); +} + +class PythonExtension implements BuildExtension { + public readonly name = "PythonExtension"; + + constructor(private options: PythonOptions = {}) { + assert( + !(this.options.requirements && this.options.requirementsFile), + "Cannot specify both requirements and requirementsFile" + ); + + if (this.options.requirementsFile) { + this.options.requirements = fs + .readFileSync(this.options.requirementsFile, "utf-8") + .split("\n"); + } + } + + async onBuildComplete(context: BuildContext, manifest: BuildManifest) { + if (context.target === "dev") { + if (this.options.pythonBinaryPath) { + process.env.PYTHON_BIN_PATH = this.options.pythonBinaryPath; + } + + return; + } + + context.logger.debug(`Adding ${this.name} to the build`); + + context.addLayer({ + id: "python-extension", + build: { + env: { + REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "", + }, + }, + image: { + instructions: ` + # Install Python + RUN apt-get update && apt-get install -y --no-install-recommends \ + python3 python3-pip python3-venv && \ + apt-get clean && rm -rf /var/lib/apt/lists/* + + # Set up Python environment + RUN python3 -m venv /opt/venv + ENV PATH="/opt/venv/bin:$PATH" + + ARG REQUIREMENTS_CONTENT + RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt + + # Install dependenciess + RUN pip install --no-cache-dir -r requirements.txt + `.split("\n"), + }, + deploy: { + env: { + PYTHON_BIN_PATH: `/opt/venv/bin/python`, + }, + override: true, + }, + }); + } +} + +export const run = async ( + args?: string, + options: Parameters[1] = {} +) => { + const cmd = `${process.env.PYTHON_BIN_PATH || "python"} ${args}`; + + logger.debug( + `Running python:\t${cmd} ${options.input ? `(with stdin)` : ""}`, + options + ); + + const result = await $({ + shell: true, + ...options, + })`${cmd}`; + + try { + assert(!result.failed, `Command failed: ${result.stderr}`); + assert(result.exitCode === 0, `Non-zero exit code: ${result.exitCode}`); + } catch (e) { + logger.error(e.message, result); + throw e; + } + + return result; +}; + +export default run; From 987ab358447cd5e8586f775a5c821076aafa6ffd Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 19:29:38 +0100 Subject: [PATCH 02/29] remove potential shell injection risk --- packages/build/src/extensions/python.ts | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 1faef7ca7f..ff6cffa5ab 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -86,21 +86,18 @@ class PythonExtension implements BuildExtension { } } -export const run = async ( - args?: string, - options: Parameters[1] = {} -) => { - const cmd = `${process.env.PYTHON_BIN_PATH || "python"} ${args}`; +export const run = async (scriptArgs: string[] = [], options: Parameters[1] = {}) => { + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; logger.debug( - `Running python:\t${cmd} ${options.input ? `(with stdin)` : ""}`, + `Running ${pythonBin} \t${JSON.stringify(scriptArgs)} ${options.input ? `(with stdin)` : ""}`, options ); const result = await $({ shell: true, ...options, - })`${cmd}`; + })(pythonBin, ...scriptArgs); try { assert(!result.failed, `Command failed: ${result.stderr}`); From 8b97cec7c0f1630ff17472f99f673d6de19f0ba3 Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 21:15:18 +0100 Subject: [PATCH 03/29] Filter out blank lines or comment lines --- packages/build/src/extensions/python.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index ff6cffa5ab..b534540321 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -20,6 +20,12 @@ export type PythonOptions = { pythonBinaryPath?: string; }; +const splitAndCleanComments = (str: string) => + str + .split("\n") + .map((line) => line.trim()) + .filter((line) => line && !line.startsWith("#")); + export function pythonExtension(options: PythonOptions = {}): BuildExtension { return new PythonExtension(options); } @@ -34,9 +40,9 @@ class PythonExtension implements BuildExtension { ); if (this.options.requirementsFile) { - this.options.requirements = fs - .readFileSync(this.options.requirementsFile, "utf-8") - .split("\n"); + this.options.requirements = splitAndCleanComments( + fs.readFileSync(this.options.requirementsFile, "utf-8") + ); } } @@ -59,7 +65,7 @@ class PythonExtension implements BuildExtension { }, }, image: { - instructions: ` + instructions: splitAndCleanComments(` # Install Python RUN apt-get update && apt-get install -y --no-install-recommends \ python3 python3-pip python3-venv && \ @@ -74,7 +80,7 @@ class PythonExtension implements BuildExtension { # Install dependenciess RUN pip install --no-cache-dir -r requirements.txt - `.split("\n"), + `), }, deploy: { env: { From 84699e50ee6d3fad5e1e5df0f4272d885c59c45f Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 21:16:09 +0100 Subject: [PATCH 04/29] fix spelling --- packages/build/src/extensions/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index b534540321..725ac46e52 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -78,7 +78,7 @@ class PythonExtension implements BuildExtension { ARG REQUIREMENTS_CONTENT RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt - # Install dependenciess + # Install dependencies RUN pip install --no-cache-dir -r requirements.txt `), }, From d00d147e54d6b7858efe50b0246ae8411414b735 Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 21:16:32 +0100 Subject: [PATCH 05/29] add pythonExtension's `runInline` --- packages/build/src/extensions/python.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 725ac46e52..16fbad4ee2 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -116,4 +116,9 @@ export const run = async (scriptArgs: string[] = [], options: Parameters[1] = {}) => { + assert(scriptContent, "Script content is required"); + return run([""], { input: scriptContent, ...options }); +}; + +export default { run, runInline }; From 500d82b3d64fc3a6bdfd7c237393d9ef5db550e3 Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 21:18:56 +0100 Subject: [PATCH 06/29] =?UTF-8?q?changes=20to=20requirements=20don?= =?UTF-8?q?=E2=80=99t=20invalidate=20the=20entire=20install=20layer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/build/src/extensions/python.ts | 28 +++++++++++++++++-------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 16fbad4ee2..fdf42aaee2 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -58,12 +58,7 @@ class PythonExtension implements BuildExtension { context.logger.debug(`Adding ${this.name} to the build`); context.addLayer({ - id: "python-extension", - build: { - env: { - REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "", - }, - }, + id: "python-installation", image: { instructions: splitAndCleanComments(` # Install Python @@ -74,7 +69,25 @@ class PythonExtension implements BuildExtension { # Set up Python environment RUN python3 -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" + `), + }, + deploy: { + env: { + PYTHON_BIN_PATH: `/opt/venv/bin/python`, + }, + override: true, + }, + }); + context.addLayer({ + id: "python-dependencies", + build: { + env: { + REQUIREMENTS_CONTENT: this.options.requirements?.join("\n") || "", + }, + }, + image: { + instructions: splitAndCleanComments(` ARG REQUIREMENTS_CONTENT RUN echo "$REQUIREMENTS_CONTENT" > requirements.txt @@ -83,9 +96,6 @@ class PythonExtension implements BuildExtension { `), }, deploy: { - env: { - PYTHON_BIN_PATH: `/opt/venv/bin/python`, - }, override: true, }, }); From f0b30d6c70a827cc6af129334f78755fdc27840c Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 22:06:49 +0100 Subject: [PATCH 07/29] copy script files on-demand --- packages/build/src/extensions/python.ts | 33 ++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 4 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index fdf42aaee2..6b292850c2 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -1,6 +1,7 @@ import fs from "node:fs"; -import { $ } from "execa"; +import { $, execa } from "execa"; import { assert } from "@std/assert"; +import { additionalFiles } from "@trigger.dev/build/extensions/core"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; import { logger } from "@trigger.dev/sdk/v3"; @@ -18,6 +19,13 @@ export type PythonOptions = { * Example: `/usr/bin/python3` or `C:\\Python39\\python.exe` */ pythonBinaryPath?: string; + /** + * An array of glob patterns that specify which Python scripts are allowed to be executed. + * + * @remarks + * These scripts will be copied to the container during the build process. + */ + scripts?: string[]; }; const splitAndCleanComments = (str: string) => @@ -47,6 +55,10 @@ class PythonExtension implements BuildExtension { } async onBuildComplete(context: BuildContext, manifest: BuildManifest) { + await additionalFiles({ + files: this.options.scripts ?? [], + }).onBuildComplete!(context, manifest); + if (context.target === "dev") { if (this.options.pythonBinaryPath) { process.env.PYTHON_BIN_PATH = this.options.pythonBinaryPath; @@ -110,10 +122,11 @@ export const run = async (scriptArgs: string[] = [], options: Parameters logger.debug(verboseLine, verboseObject), ...options, - })(pythonBin, ...scriptArgs); + })(pythonBin, scriptArgs); try { assert(!result.failed, `Command failed: ${result.stderr}`); @@ -126,9 +139,21 @@ export const run = async (scriptArgs: string[] = [], options: Parameters[1] = {} +) => { + assert(scriptPath, "Script path is required"); + assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); + + return run([scriptPath, ...scriptArgs], options); +}; + export const runInline = (scriptContent: string, options: Parameters[1] = {}) => { assert(scriptContent, "Script content is required"); + return run([""], { input: scriptContent, ...options }); }; -export default { run, runInline }; +export default { run, runScript, runInline }; From b63554e2ebd60dd10594bc83ffedc7d102b26e10 Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 22:19:19 +0100 Subject: [PATCH 08/29] improve PythonExtension types and logging --- packages/build/src/extensions/python.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 6b292850c2..5b88f718c4 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -1,11 +1,13 @@ import fs from "node:fs"; -import { $, execa } from "execa"; +import { execa } from "execa"; import { assert } from "@std/assert"; import { additionalFiles } from "@trigger.dev/build/extensions/core"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; import { logger } from "@trigger.dev/sdk/v3"; +import type { VerboseObject } from "execa"; + export type PythonOptions = { requirements?: string[]; requirementsFile?: string; @@ -28,6 +30,8 @@ export type PythonOptions = { scripts?: string[]; }; +type ExecaOptions = Parameters[1]; + const splitAndCleanComments = (str: string) => str .split("\n") @@ -114,17 +118,13 @@ class PythonExtension implements BuildExtension { } } -export const run = async (scriptArgs: string[] = [], options: Parameters[1] = {}) => { +export const run = async (scriptArgs: string[] = [], options: ExecaOptions = {}) => { const pythonBin = process.env.PYTHON_BIN_PATH || "python"; - logger.debug( - `Running ${pythonBin} \t${JSON.stringify(scriptArgs)} ${options.input ? `(with stdin)` : ""}`, - options - ); - const result = await execa({ shell: true, - verbose: (verboseLine, verboseObject) => logger.debug(verboseLine, verboseObject), + verbose: (verboseLine: string, verboseObject: VerboseObject) => + logger.debug(verboseObject.message, verboseObject), ...options, })(pythonBin, scriptArgs); @@ -142,7 +142,7 @@ export const run = async (scriptArgs: string[] = [], options: Parameters[1] = {} + options: ExecaOptions = {} ) => { assert(scriptPath, "Script path is required"); assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); @@ -150,7 +150,7 @@ export const runScript = ( return run([scriptPath, ...scriptArgs], options); }; -export const runInline = (scriptContent: string, options: Parameters[1] = {}) => { +export const runInline = (scriptContent: string, options: ExecaOptions = {}) => { assert(scriptContent, "Script content is required"); return run([""], { input: scriptContent, ...options }); From 8ad700970315f3fd18e0df687270918a4652f115 Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 9 Feb 2025 22:48:06 +0100 Subject: [PATCH 09/29] add changeset --- .changeset/little-trains-begin.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/little-trains-begin.md diff --git a/.changeset/little-trains-begin.md b/.changeset/little-trains-begin.md new file mode 100644 index 0000000000..89b8998251 --- /dev/null +++ b/.changeset/little-trains-begin.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/build": minor +--- + +Introduced a new Python extension to enhance the build process. It now allows users to execute Python scripts with improved support and error handling. From c3cbd6137813c442011e57d60a6b9fb9f0d1e3b5 Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 02:32:12 +0100 Subject: [PATCH 10/29] fix broken imports --- packages/build/package.json | 2 ++ packages/build/src/extensions/python.ts | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/build/package.json b/packages/build/package.json index a697c62873..4b73cc9f8b 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -66,6 +66,8 @@ }, "dependencies": { "@trigger.dev/core": "workspace:3.3.13", + "@trigger.dev/sdk": "workspace:3.3.13", + "execa": "^9.5.2", "pkg-types": "^1.1.3", "tinyglobby": "^0.2.2", "tsconfck": "3.1.3" diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 5b88f718c4..c06e2803bd 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -1,6 +1,6 @@ import fs from "node:fs"; +import assert from "node:assert"; import { execa } from "execa"; -import { assert } from "@std/assert"; import { additionalFiles } from "@trigger.dev/build/extensions/core"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; From be2cefe5051403340f703ee65bed8736b7d86467 Mon Sep 17 00:00:00 2001 From: Victor Duarte Date: Mon, 10 Feb 2025 11:42:43 +0100 Subject: [PATCH 11/29] Improve security of inline script execution Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- packages/build/src/extensions/python.ts | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index c06e2803bd..4363c08603 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -150,10 +150,19 @@ export const runScript = ( return run([scriptPath, ...scriptArgs], options); }; -export const runInline = (scriptContent: string, options: ExecaOptions = {}) => { +export const runInline = async (scriptContent: string, options: ExecaOptions = {}) => { assert(scriptContent, "Script content is required"); - return run([""], { input: scriptContent, ...options }); + // Create a temporary file with restricted permissions + const tmpFile = `/tmp/script_${Date.now()}.py`; + await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); + + try { + return await runScript(tmpFile, [], options); + } finally { + // Clean up temporary file + await fs.promises.unlink(tmpFile); + } }; export default { run, runScript, runInline }; From bb59bf8e8e5d4fc9df6d69714e592ee4cd3e9b8c Mon Sep 17 00:00:00 2001 From: Victor Duarte Date: Mon, 10 Feb 2025 11:52:51 +0100 Subject: [PATCH 12/29] Add file existence check for requirementsFile Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- packages/build/src/extensions/python.ts | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 4363c08603..5a7f81dff3 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -52,6 +52,10 @@ class PythonExtension implements BuildExtension { ); if (this.options.requirementsFile) { + assert( + fs.existsSync(this.options.requirementsFile), + `Requirements file not found: ${this.options.requirementsFile}` + ); this.options.requirements = splitAndCleanComments( fs.readFileSync(this.options.requirementsFile, "utf-8") ); From 65d84d070c691f98d5c52d2b0eb75f5eda561b6d Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 11:54:52 +0100 Subject: [PATCH 13/29] update lock file --- pnpm-lock.yaml | 99 +++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 604337f9da..e06d1b8041 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1038,6 +1038,12 @@ importers: '@trigger.dev/core': specifier: workspace:3.3.13 version: link:../core + '@trigger.dev/sdk': + specifier: workspace:3.3.13 + version: link:../trigger-sdk + execa: + specifier: ^9.5.2 + version: 9.5.2 pkg-types: specifier: ^1.1.3 version: 1.1.3 @@ -15018,6 +15024,10 @@ packages: resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==} dev: true + /@sec-ant/readable-stream@0.4.1: + resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} + dev: false + /@selderee/plugin-htmlparser2@0.11.0: resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==} dependencies: @@ -15176,6 +15186,11 @@ packages: resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} + /@sindresorhus/merge-streams@4.0.0: + resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} + engines: {node: '>=18'} + dev: false + /@sindresorhus/slugify@2.2.1: resolution: {integrity: sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==} engines: {node: '>=12'} @@ -22011,6 +22026,24 @@ packages: signal-exit: 4.1.0 strip-final-newline: 3.0.0 + /execa@9.5.2: + resolution: {integrity: sha512-EHlpxMCpHWSAh1dgS6bVeoLAXGnJNdR93aabr4QCGbzOM73o5XmRfM/e5FUqsw3aagP8S8XEWUWFAxnRBnAF0Q==} + engines: {node: ^18.19.0 || >=20.5.0} + dependencies: + '@sindresorhus/merge-streams': 4.0.0 + cross-spawn: 7.0.3 + figures: 6.1.0 + get-stream: 9.0.1 + human-signals: 8.0.0 + is-plain-obj: 4.1.0 + is-stream: 4.0.1 + npm-run-path: 6.0.0 + pretty-ms: 9.2.0 + signal-exit: 4.1.0 + strip-final-newline: 4.0.0 + yoctocolors: 2.1.1 + dev: false + /exit-hook@2.2.1: resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} engines: {node: '>=6'} @@ -22242,6 +22275,13 @@ packages: escape-string-regexp: 1.0.5 dev: false + /figures@6.1.0: + resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} + engines: {node: '>=18'} + dependencies: + is-unicode-supported: 2.1.0 + dev: false + /file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -22670,6 +22710,14 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} + /get-stream@9.0.1: + resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} + engines: {node: '>=18'} + dependencies: + '@sec-ant/readable-stream': 0.4.1 + is-stream: 4.0.1 + dev: false + /get-symbol-description@1.0.0: resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} engines: {node: '>= 0.4'} @@ -23279,6 +23327,11 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} + /human-signals@8.0.0: + resolution: {integrity: sha512-/1/GPCpDUCCYwlERiYjxoczfP0zfvZMU/OWgQPMya9AbAE24vseigFdhAMObpc8Q4lc/kjutPfUddDYyAmejnA==} + engines: {node: '>=18.18.0'} + dev: false + /humanize-duration@3.27.3: resolution: {integrity: sha512-iimHkHPfIAQ8zCDQLgn08pRqSVioyWvnGfaQ8gond2wf7Jq2jJ+24ykmnRyiz3fIldcn4oUuQXpjqKLhSVR7lw==} dev: false @@ -23739,7 +23792,6 @@ packages: /is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} - dev: true /is-plain-object@5.0.0: resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==} @@ -23784,6 +23836,11 @@ packages: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} + /is-stream@4.0.1: + resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} + engines: {node: '>=18'} + dev: false + /is-string@1.0.7: resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} engines: {node: '>= 0.4'} @@ -23828,6 +23885,11 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} + /is-unicode-supported@2.1.0: + resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} + engines: {node: '>=18'} + dev: false + /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -25866,6 +25928,14 @@ packages: dependencies: path-key: 4.0.0 + /npm-run-path@6.0.0: + resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} + engines: {node: '>=18'} + dependencies: + path-key: 4.0.0 + unicorn-magic: 0.3.0 + dev: false + /num2fraction@1.2.2: resolution: {integrity: sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg==} dev: false @@ -26494,6 +26564,11 @@ packages: engines: {node: '>=6'} dev: true + /parse-ms@4.0.0: + resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} + engines: {node: '>=18'} + dev: false + /parse5-htmlparser2-tree-adapter@6.0.1: resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} dependencies: @@ -27287,6 +27362,13 @@ packages: parse-ms: 2.1.0 dev: true + /pretty-ms@9.2.0: + resolution: {integrity: sha512-4yf0QO/sllf/1zbZWYnvWw3NxCQwLXKzIj0G849LSufP15BXKM0rbD2Z3wVnkMfjdn/CB0Dpp444gYAACdsplg==} + engines: {node: '>=18'} + dependencies: + parse-ms: 4.0.0 + dev: false + /printable-characters@1.0.42: resolution: {integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==} @@ -29779,6 +29861,11 @@ packages: resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} engines: {node: '>=12'} + /strip-final-newline@4.0.0: + resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} + engines: {node: '>=18'} + dev: false + /strip-indent@3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} @@ -31264,6 +31351,11 @@ packages: engines: {node: '>=18'} dev: true + /unicorn-magic@0.3.0: + resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} + engines: {node: '>=18'} + dev: false + /unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} dependencies: @@ -32867,6 +32959,11 @@ packages: engines: {node: '>=12.20'} dev: false + /yoctocolors@2.1.1: + resolution: {integrity: sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==} + engines: {node: '>=18'} + dev: false + /youch@3.3.3: resolution: {integrity: sha512-qSFXUk3UZBLfggAW3dJKg0BMblG5biqSF8M34E06o5CSsZtH92u9Hqmj2RzGiHDi64fhe83+4tENFP2DB6t6ZA==} dependencies: From 7c969302ce43e8c10aff95ac8828ae18e5a1ac37 Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 12:27:51 +0100 Subject: [PATCH 14/29] Enhance error handling with detailed error information --- packages/build/src/extensions/python.ts | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 5a7f81dff3..96322ffa49 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -133,10 +133,19 @@ export const run = async (scriptArgs: string[] = [], options: ExecaOptions = {}) })(pythonBin, scriptArgs); try { - assert(!result.failed, `Command failed: ${result.stderr}`); - assert(result.exitCode === 0, `Non-zero exit code: ${result.exitCode}`); + assert(!result.failed, `Python command failed: ${result.stderr}\nCommand: ${result.command}`); + assert( + result.exitCode === 0, + `Python command exited with non-zero code ${result.exitCode}\nStdout: ${result.stdout}\nStderr: ${result.stderr}` + ); } catch (e) { - logger.error(e.message, result); + logger.error("Python command execution failed", { + error: e.message, + command: result.command, + stdout: result.stdout, + stderr: result.stderr, + exitCode: result.exitCode, + }); throw e; } @@ -160,7 +169,7 @@ export const runInline = async (scriptContent: string, options: ExecaOptions = { // Create a temporary file with restricted permissions const tmpFile = `/tmp/script_${Date.now()}.py`; await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); - + try { return await runScript(tmpFile, [], options); } finally { From c599809973566b44f683f5d844a8e7f23886bde2 Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 12:38:32 +0100 Subject: [PATCH 15/29] Add portable type annotation --- packages/build/src/extensions/python.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 96322ffa49..477f921740 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -122,7 +122,10 @@ class PythonExtension implements BuildExtension { } } -export const run = async (scriptArgs: string[] = [], options: ExecaOptions = {}) => { +export const run = async ( + scriptArgs: string[] = [], + options: ExecaOptions = {} +): Promise> => { const pythonBin = process.env.PYTHON_BIN_PATH || "python"; const result = await execa({ From 015f3aaa79ceb5396709c6b23e7c1dae18588541 Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 12:49:25 +0100 Subject: [PATCH 16/29] fix error TS18046: 'e' is of type 'unknown' --- packages/build/src/extensions/python.ts | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 477f921740..525114ea1b 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -130,8 +130,7 @@ export const run = async ( const result = await execa({ shell: true, - verbose: (verboseLine: string, verboseObject: VerboseObject) => - logger.debug(verboseObject.message, verboseObject), + verbose: (line: string, obj: VerboseObject) => logger.debug(obj.message, obj), ...options, })(pythonBin, scriptArgs); @@ -141,15 +140,15 @@ export const run = async ( result.exitCode === 0, `Python command exited with non-zero code ${result.exitCode}\nStdout: ${result.stdout}\nStderr: ${result.stderr}` ); - } catch (e) { + } catch (error) { logger.error("Python command execution failed", { - error: e.message, + error: error instanceof Error ? error.message : error, command: result.command, stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, }); - throw e; + throw error; } return result; From 4302077e46a4753449563e5c4e1dd0f502c2fb55 Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 13:00:43 +0100 Subject: [PATCH 17/29] export the python extension --- packages/build/package.json | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/packages/build/package.json b/packages/build/package.json index 4b73cc9f8b..18e0b996f3 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -28,6 +28,7 @@ "./extensions/prisma": "./src/extensions/prisma.ts", "./extensions/audioWaveform": "./src/extensions/audioWaveform.ts", "./extensions/typescript": "./src/extensions/typescript.ts", + "./extensions/python": "./src/extensions/python.ts", "./extensions/puppeteer": "./src/extensions/puppeteer.ts" }, "sourceDialects": [ @@ -51,6 +52,9 @@ "extensions/typescript": [ "dist/commonjs/extensions/typescript.d.ts" ], + "extensions/python": [ + "dist/commonjs/extensions/python.d.ts" + ], "extensions/puppeteer": [ "dist/commonjs/extensions/puppeteer.d.ts" ] @@ -152,6 +156,17 @@ "default": "./dist/commonjs/extensions/typescript.js" } }, + "./extensions/python": { + "import": { + "@triggerdotdev/source": "./src/extensions/python.ts", + "types": "./dist/esm/extensions/python.d.ts", + "default": "./dist/esm/extensions/python.js" + }, + "require": { + "types": "./dist/commonjs/extensions/python.d.ts", + "default": "./dist/commonjs/extensions/python.js" + } + }, "./extensions/puppeteer": { "import": { "@triggerdotdev/source": "./src/extensions/puppeteer.ts", From 8ef78ee365d8c13bc1d175ad604655be9f97a846 Mon Sep 17 00:00:00 2001 From: zvictor Date: Mon, 10 Feb 2025 13:00:59 +0100 Subject: [PATCH 18/29] add `pythonExtension` to the catalog --- references/v3-catalog/trigger.config.ts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/references/v3-catalog/trigger.config.ts b/references/v3-catalog/trigger.config.ts index 830ac38999..c6fcca68e3 100644 --- a/references/v3-catalog/trigger.config.ts +++ b/references/v3-catalog/trigger.config.ts @@ -7,6 +7,7 @@ import { ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core"; import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; +import { pythonExtension } from "@trigger.dev/build/extensions/python"; import { defineConfig } from "@trigger.dev/sdk/v3"; export { handleError } from "./src/handleError.js"; @@ -86,6 +87,7 @@ export default defineConfig({ value: secret.secretValue, })); }), + pythonExtension(), puppeteer(), ], external: ["re2"], From ca51709a48684f3fbf4629f8044a03505fe9e49e Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 11 Feb 2025 02:13:19 +0100 Subject: [PATCH 19/29] fix `Cannot find module '@trigger.dev/build/extensions/core' (TS2307) --- packages/build/src/extensions/python.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 525114ea1b..2303cb0c65 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import assert from "node:assert"; import { execa } from "execa"; -import { additionalFiles } from "@trigger.dev/build/extensions/core"; +import { additionalFiles } from "./core/additionalFiles.js"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; import { logger } from "@trigger.dev/sdk/v3"; From ddc706abd83e6e9a76540650831d29d07ecf71c2 Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 11 Feb 2025 03:11:14 +0100 Subject: [PATCH 20/29] replace execa by tinyexec --- packages/build/package.json | 2 +- packages/build/src/extensions/python.ts | 26 +++--- pnpm-lock.yaml | 103 ++---------------------- 3 files changed, 18 insertions(+), 113 deletions(-) diff --git a/packages/build/package.json b/packages/build/package.json index 18e0b996f3..1a14eb84f7 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -71,8 +71,8 @@ "dependencies": { "@trigger.dev/core": "workspace:3.3.13", "@trigger.dev/sdk": "workspace:3.3.13", - "execa": "^9.5.2", "pkg-types": "^1.1.3", + "tinyexec": "^0.3.2", "tinyglobby": "^0.2.2", "tsconfck": "3.1.3" }, diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 2303cb0c65..74961fa23b 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -1,12 +1,10 @@ import fs from "node:fs"; import assert from "node:assert"; -import { execa } from "execa"; import { additionalFiles } from "./core/additionalFiles.js"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; import { logger } from "@trigger.dev/sdk/v3"; - -import type { VerboseObject } from "execa"; +import { x, Options as XOptions, Result } from "tinyexec"; export type PythonOptions = { requirements?: string[]; @@ -30,8 +28,6 @@ export type PythonOptions = { scripts?: string[]; }; -type ExecaOptions = Parameters[1]; - const splitAndCleanComments = (str: string) => str .split("\n") @@ -124,18 +120,16 @@ class PythonExtension implements BuildExtension { export const run = async ( scriptArgs: string[] = [], - options: ExecaOptions = {} -): Promise> => { + options: Partial = {} +): Promise => { const pythonBin = process.env.PYTHON_BIN_PATH || "python"; - const result = await execa({ - shell: true, - verbose: (line: string, obj: VerboseObject) => logger.debug(obj.message, obj), + const result = await x(pythonBin, scriptArgs, { ...options, - })(pythonBin, scriptArgs); + throwOnError: false, // Ensure errors are handled manually + }); try { - assert(!result.failed, `Python command failed: ${result.stderr}\nCommand: ${result.command}`); assert( result.exitCode === 0, `Python command exited with non-zero code ${result.exitCode}\nStdout: ${result.stdout}\nStderr: ${result.stderr}` @@ -143,7 +137,7 @@ export const run = async ( } catch (error) { logger.error("Python command execution failed", { error: error instanceof Error ? error.message : error, - command: result.command, + command: `${pythonBin} ${scriptArgs.join(" ")}`, stdout: result.stdout, stderr: result.stderr, exitCode: result.exitCode, @@ -157,7 +151,7 @@ export const run = async ( export const runScript = ( scriptPath: string, scriptArgs: string[] = [], - options: ExecaOptions = {} + options: Partial = {} ) => { assert(scriptPath, "Script path is required"); assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); @@ -165,17 +159,15 @@ export const runScript = ( return run([scriptPath, ...scriptArgs], options); }; -export const runInline = async (scriptContent: string, options: ExecaOptions = {}) => { +export const runInline = async (scriptContent: string, options: Partial = {}) => { assert(scriptContent, "Script content is required"); - // Create a temporary file with restricted permissions const tmpFile = `/tmp/script_${Date.now()}.py`; await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); try { return await runScript(tmpFile, [], options); } finally { - // Clean up temporary file await fs.promises.unlink(tmpFile); } }; diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index e06d1b8041..59c21c28ae 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1041,12 +1041,12 @@ importers: '@trigger.dev/sdk': specifier: workspace:3.3.13 version: link:../trigger-sdk - execa: - specifier: ^9.5.2 - version: 9.5.2 pkg-types: specifier: ^1.1.3 version: 1.1.3 + tinyexec: + specifier: ^0.3.2 + version: 0.3.2 tinyglobby: specifier: ^0.2.2 version: 0.2.2 @@ -15024,10 +15024,6 @@ packages: resolution: {integrity: sha512-sXo/qW2/pAcmT43VoRKOJbDOfV3cYpq3szSVfIThQXNt+E4DfKj361vaAt3c88U5tPUxzEswam7GW48PJqtKAg==} dev: true - /@sec-ant/readable-stream@0.4.1: - resolution: {integrity: sha512-831qok9r2t8AlxLko40y2ebgSDhenenCatLVeW/uBtnHPyhHOvG0C7TvfgecV+wHzIm5KUICgzmVpWS+IMEAeg==} - dev: false - /@selderee/plugin-htmlparser2@0.11.0: resolution: {integrity: sha512-P33hHGdldxGabLFjPPpaTxVolMrzrcegejx+0GxjrIb9Zv48D8yAIA/QTDR2dFl7Uz7urX8aX6+5bCZslr+gWQ==} dependencies: @@ -15186,11 +15182,6 @@ packages: resolution: {integrity: sha512-t09vSN3MdfsyCHoFcTRCH/iUtG7OJ0CsjzB8cjAmKc/va/kIgeDI/TxsigdncE/4be734m0cvIYwNaV4i2XqAw==} engines: {node: '>=10'} - /@sindresorhus/merge-streams@4.0.0: - resolution: {integrity: sha512-tlqY9xq5ukxTUZBmoOp+m61cqwQD5pHJtFY3Mn8CA8ps6yghLH/Hw8UPdqg4OLmFW3IFlcXnQNmo/dh8HzXYIQ==} - engines: {node: '>=18'} - dev: false - /@sindresorhus/slugify@2.2.1: resolution: {integrity: sha512-MkngSCRZ8JdSOCHRaYd+D01XhvU3Hjy6MGl06zhOk614hp9EOAp5gIkBeQg7wtmxpitU6eAL4kdiRMcJa2dlrw==} engines: {node: '>=12'} @@ -22026,24 +22017,6 @@ packages: signal-exit: 4.1.0 strip-final-newline: 3.0.0 - /execa@9.5.2: - resolution: {integrity: sha512-EHlpxMCpHWSAh1dgS6bVeoLAXGnJNdR93aabr4QCGbzOM73o5XmRfM/e5FUqsw3aagP8S8XEWUWFAxnRBnAF0Q==} - engines: {node: ^18.19.0 || >=20.5.0} - dependencies: - '@sindresorhus/merge-streams': 4.0.0 - cross-spawn: 7.0.3 - figures: 6.1.0 - get-stream: 9.0.1 - human-signals: 8.0.0 - is-plain-obj: 4.1.0 - is-stream: 4.0.1 - npm-run-path: 6.0.0 - pretty-ms: 9.2.0 - signal-exit: 4.1.0 - strip-final-newline: 4.0.0 - yoctocolors: 2.1.1 - dev: false - /exit-hook@2.2.1: resolution: {integrity: sha512-eNTPlAD67BmP31LDINZ3U7HSF8l57TxOY2PmBJ1shpCvpnxBF93mWCE8YHBnXs8qiUZJc9WDcWIeC3a2HIAMfw==} engines: {node: '>=6'} @@ -22275,13 +22248,6 @@ packages: escape-string-regexp: 1.0.5 dev: false - /figures@6.1.0: - resolution: {integrity: sha512-d+l3qxjSesT4V7v2fh+QnmFnUWv9lSpjarhShNTgBOfA0ttejbQUAlHLitbjkoRiDulW0OPoQPYIGhIC8ohejg==} - engines: {node: '>=18'} - dependencies: - is-unicode-supported: 2.1.0 - dev: false - /file-entry-cache@6.0.1: resolution: {integrity: sha512-7Gps/XWymbLk2QLYK4NzpMOrYjMhdIxXuIvy2QBsLE6ljuodKvdkWs/cpyJJ3CVIVpH0Oi1Hvg1ovbMzLdFBBg==} engines: {node: ^10.12.0 || >=12.0.0} @@ -22710,14 +22676,6 @@ packages: resolution: {integrity: sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA==} engines: {node: '>=16'} - /get-stream@9.0.1: - resolution: {integrity: sha512-kVCxPF3vQM/N0B1PmoqVUqgHP+EeVjmZSQn+1oCRPxd2P21P2F19lIgbR3HBosbB1PUhOAoctJnfEn2GbN2eZA==} - engines: {node: '>=18'} - dependencies: - '@sec-ant/readable-stream': 0.4.1 - is-stream: 4.0.1 - dev: false - /get-symbol-description@1.0.0: resolution: {integrity: sha512-2EmdH1YvIQiZpltCNgkuiUnyukzxM/R6NDJX31Ke3BG1Nq5b0S2PhX59UKi9vZpPDQVdqn+1IcaAwnzTT5vCjw==} engines: {node: '>= 0.4'} @@ -23327,11 +23285,6 @@ packages: resolution: {integrity: sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ==} engines: {node: '>=16.17.0'} - /human-signals@8.0.0: - resolution: {integrity: sha512-/1/GPCpDUCCYwlERiYjxoczfP0zfvZMU/OWgQPMya9AbAE24vseigFdhAMObpc8Q4lc/kjutPfUddDYyAmejnA==} - engines: {node: '>=18.18.0'} - dev: false - /humanize-duration@3.27.3: resolution: {integrity: sha512-iimHkHPfIAQ8zCDQLgn08pRqSVioyWvnGfaQ8gond2wf7Jq2jJ+24ykmnRyiz3fIldcn4oUuQXpjqKLhSVR7lw==} dev: false @@ -23792,6 +23745,7 @@ packages: /is-plain-obj@4.1.0: resolution: {integrity: sha512-+Pgi+vMuUNkJyExiMBt5IlFoMyKnr5zhJ4Uspz58WOhBF5QoIZkFyNHIbBAtHwzVAgk5RtndVNsDRN61/mmDqg==} engines: {node: '>=12'} + dev: true /is-plain-object@5.0.0: resolution: {integrity: sha512-VRSzKkbMm5jMDoKLbltAkFQ5Qr7VDiTFGXxYFXXowVj387GeGNOCsOH6Msy00SGZ3Fp84b1Naa1psqgcCIEP5Q==} @@ -23836,11 +23790,6 @@ packages: resolution: {integrity: sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA==} engines: {node: ^12.20.0 || ^14.13.1 || >=16.0.0} - /is-stream@4.0.1: - resolution: {integrity: sha512-Dnz92NInDqYckGEUJv689RbRiTSEHCQ7wOVeALbkOz999YpqT46yMRIGtSNl2iCL1waAZSx40+h59NV/EwzV/A==} - engines: {node: '>=18'} - dev: false - /is-string@1.0.7: resolution: {integrity: sha512-tE2UXzivje6ofPW7l23cjDOMa09gb7xlAqG6jG5ej6uPV32TlWP3NKPigtaGeHNu9fohccRYvIiZMfOOnOYUtg==} engines: {node: '>= 0.4'} @@ -23885,11 +23834,6 @@ packages: resolution: {integrity: sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==} engines: {node: '>=10'} - /is-unicode-supported@2.1.0: - resolution: {integrity: sha512-mE00Gnza5EEB3Ds0HfMyllZzbBrmLOX3vfWoj9A9PEnTfratQ/BcaJOuMhnkhjXvb2+FkY3VuHqtAGpTPmglFQ==} - engines: {node: '>=18'} - dev: false - /is-weakref@1.0.2: resolution: {integrity: sha512-qctsuLZmIQ0+vSSMfoVvyFe2+GSEvnmZ2ezTup1SBse9+twCCeial6EEi3Nc2KFcf6+qz2FBPnjXsk8xhKSaPQ==} dependencies: @@ -25928,14 +25872,6 @@ packages: dependencies: path-key: 4.0.0 - /npm-run-path@6.0.0: - resolution: {integrity: sha512-9qny7Z9DsQU8Ou39ERsPU4OZQlSTP47ShQzuKZ6PRXpYLtIFgl/DEBYEXKlvcEa+9tHVcK8CF81Y2V72qaZhWA==} - engines: {node: '>=18'} - dependencies: - path-key: 4.0.0 - unicorn-magic: 0.3.0 - dev: false - /num2fraction@1.2.2: resolution: {integrity: sha512-Y1wZESM7VUThYY+4W+X4ySH2maqcA+p7UR+w8VWNWVAd6lwuXXWz/w/Cz43J/dI2I+PS6wD5N+bJUF+gjWvIqg==} dev: false @@ -26564,11 +26500,6 @@ packages: engines: {node: '>=6'} dev: true - /parse-ms@4.0.0: - resolution: {integrity: sha512-TXfryirbmq34y8QBwgqCVLi+8oA3oWx2eAnSn62ITyEhEYaWRlVZ2DvMM9eZbMs/RfxPu/PK/aBLyGj4IrqMHw==} - engines: {node: '>=18'} - dev: false - /parse5-htmlparser2-tree-adapter@6.0.1: resolution: {integrity: sha512-qPuWvbLgvDGilKc5BoicRovlT4MtYT6JfJyBOMDsKoiT+GiuP5qyrPCnR9HcPECIJJmZh5jRndyNThnhhb/vlA==} dependencies: @@ -27362,13 +27293,6 @@ packages: parse-ms: 2.1.0 dev: true - /pretty-ms@9.2.0: - resolution: {integrity: sha512-4yf0QO/sllf/1zbZWYnvWw3NxCQwLXKzIj0G849LSufP15BXKM0rbD2Z3wVnkMfjdn/CB0Dpp444gYAACdsplg==} - engines: {node: '>=18'} - dependencies: - parse-ms: 4.0.0 - dev: false - /printable-characters@1.0.42: resolution: {integrity: sha512-dKp+C4iXWK4vVYZmYSd0KBH5F/h1HoZRsbJ82AVKRO3PEo8L4lBS/vLwhVtpwwuYcoIsVY+1JYKR268yn480uQ==} @@ -29861,11 +29785,6 @@ packages: resolution: {integrity: sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw==} engines: {node: '>=12'} - /strip-final-newline@4.0.0: - resolution: {integrity: sha512-aulFJcD6YK8V1G7iRB5tigAP4TsHBZZrOV8pjV++zdUwmeV8uzbY7yn6h9MswN62adStNZFuCIx4haBnRuMDaw==} - engines: {node: '>=18'} - dev: false - /strip-indent@3.0.0: resolution: {integrity: sha512-laJTa3Jb+VQpaC6DseHhF7dXVqHTfJPCRDaEbid/drOhgitgYku/letMUqOXFoWV0zIIUbjpdH2t+tYj4bQMRQ==} engines: {node: '>=8'} @@ -30542,6 +30461,10 @@ packages: resolution: {integrity: sha512-WiCJLEECkO18gwqIp6+hJg0//p23HXp4S+gGtAKu3mI2F2/sXC4FvHvXvB0zJVVaTPhx1/tOwdbRsa1sOBIKqQ==} dev: false + /tinyexec@0.3.2: + resolution: {integrity: sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA==} + dev: false + /tinyglobby@0.2.10: resolution: {integrity: sha512-Zc+8eJlFMvgatPZTl6A9L/yht8QqdmUNtURHaKZLmKBE12hNPSrqNkUp2cs3M/UKmNVVAMFQYSjYIVHDjW5zew==} engines: {node: '>=12.0.0'} @@ -31351,11 +31274,6 @@ packages: engines: {node: '>=18'} dev: true - /unicorn-magic@0.3.0: - resolution: {integrity: sha512-+QBBXBCvifc56fsbuxZQ6Sic3wqqc3WWaqxs58gvJrcOuN83HGTCwz3oS5phzU9LthRNE9VrJCFCLUgHeeFnfA==} - engines: {node: '>=18'} - dev: false - /unified@10.1.2: resolution: {integrity: sha512-pUSWAi/RAnVy1Pif2kAoeWNBa3JVrx0MId2LASj8G+7AiHWoKZNTomq6LG326T68U7/e263X6fTdcXIy7XnF7Q==} dependencies: @@ -32959,11 +32877,6 @@ packages: engines: {node: '>=12.20'} dev: false - /yoctocolors@2.1.1: - resolution: {integrity: sha512-GQHQqAopRhwU8Kt1DDM8NjibDXHC8eoh1erhGAJPEyveY9qqVeXvVikNKrDz69sHowPMorbPUrH/mx8c50eiBQ==} - engines: {node: '>=18'} - dev: false - /youch@3.3.3: resolution: {integrity: sha512-qSFXUk3UZBLfggAW3dJKg0BMblG5biqSF8M34E06o5CSsZtH92u9Hqmj2RzGiHDi64fhe83+4tENFP2DB6t6ZA==} dependencies: From cf95fcaafbaa1730a7c6011c6c7c19b3f7c8a1ab Mon Sep 17 00:00:00 2001 From: Matt Aitken Date: Tue, 11 Feb 2025 14:17:45 +0000 Subject: [PATCH 21/29] Update pnpm-lock.yaml --- pnpm-lock.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index f0cf09d19d..a100f2e4a1 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1039,7 +1039,7 @@ importers: specifier: workspace:3.3.14 version: link:../core '@trigger.dev/sdk': - specifier: workspace:3.3.13 + specifier: workspace:3.3.14 version: link:../trigger-sdk pkg-types: specifier: ^1.1.3 From c24612006bb5c58c1aa62ce677f6eba8d5e1ef24 Mon Sep 17 00:00:00 2001 From: zvictor Date: Sun, 16 Feb 2025 11:18:18 +0100 Subject: [PATCH 22/29] add custom traces instead of logging --- packages/build/src/extensions/python.ts | 35 +++++++++++-------------- 1 file changed, 16 insertions(+), 19 deletions(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 74961fa23b..49eb4ff531 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -124,28 +124,25 @@ export const run = async ( ): Promise => { const pythonBin = process.env.PYTHON_BIN_PATH || "python"; - const result = await x(pythonBin, scriptArgs, { - ...options, - throwOnError: false, // Ensure errors are handled manually - }); - - try { - assert( - result.exitCode === 0, - `Python command exited with non-zero code ${result.exitCode}\nStdout: ${result.stdout}\nStderr: ${result.stderr}` - ); - } catch (error) { - logger.error("Python command execution failed", { - error: error instanceof Error ? error.message : error, + return await logger.trace("Python call", async (span) => { + span.addEvent("Properties", { command: `${pythonBin} ${scriptArgs.join(" ")}`, - stdout: result.stdout, - stderr: result.stderr, - exitCode: result.exitCode, }); - throw error; - } - return result; + const result = await x(pythonBin, scriptArgs, { + ...options, + throwOnError: false, // Ensure errors are handled manually + }); + + span.addEvent("Output", { ...result }); + + if (result.exitCode !== 0) { + logger.error(result.stderr, { ...result }); + throw new Error(`Python command exited with non-zero code ${result.exitCode}`); + } + + return result; + }); }; export const runScript = ( From 477663728ed90f26d1f30993fe22526b673f5942 Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 25 Feb 2025 05:34:50 +0100 Subject: [PATCH 23/29] The cleanup in the finally block does not fail silently anymore --- packages/build/src/extensions/python.ts | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/packages/build/src/extensions/python.ts b/packages/build/src/extensions/python.ts index 49eb4ff531..3caa1a4300 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/build/src/extensions/python.ts @@ -165,7 +165,13 @@ export const runInline = async (scriptContent: string, options: Partial Date: Tue, 25 Feb 2025 06:38:41 +0100 Subject: [PATCH 24/29] move python runtime/extension to independent package --- .changeset/little-trains-begin.md | 2 +- packages/build/package.json | 17 ---- packages/python/CHANGELOG.md | 1 + packages/python/LICENSE | 21 +++++ packages/python/README.md | 3 + packages/python/package.json | 93 +++++++++++++++++++ .../python.ts => python/src/extension.ts} | 63 +------------ packages/python/src/index.ts | 63 +++++++++++++ packages/python/tsconfig.json | 8 ++ packages/python/tsconfig.src.json | 10 ++ references/v3-catalog/trigger.config.ts | 2 +- 11 files changed, 203 insertions(+), 80 deletions(-) create mode 100644 packages/python/CHANGELOG.md create mode 100644 packages/python/LICENSE create mode 100644 packages/python/README.md create mode 100644 packages/python/package.json rename packages/{build/src/extensions/python.ts => python/src/extension.ts} (64%) create mode 100644 packages/python/src/index.ts create mode 100644 packages/python/tsconfig.json create mode 100644 packages/python/tsconfig.src.json diff --git a/.changeset/little-trains-begin.md b/.changeset/little-trains-begin.md index 89b8998251..15c54cbb0f 100644 --- a/.changeset/little-trains-begin.md +++ b/.changeset/little-trains-begin.md @@ -1,5 +1,5 @@ --- -"@trigger.dev/build": minor +"@trigger.dev/python": minor --- Introduced a new Python extension to enhance the build process. It now allows users to execute Python scripts with improved support and error handling. diff --git a/packages/build/package.json b/packages/build/package.json index 8f42528600..5ccdfd6305 100644 --- a/packages/build/package.json +++ b/packages/build/package.json @@ -28,7 +28,6 @@ "./extensions/prisma": "./src/extensions/prisma.ts", "./extensions/audioWaveform": "./src/extensions/audioWaveform.ts", "./extensions/typescript": "./src/extensions/typescript.ts", - "./extensions/python": "./src/extensions/python.ts", "./extensions/puppeteer": "./src/extensions/puppeteer.ts" }, "sourceDialects": [ @@ -52,9 +51,6 @@ "extensions/typescript": [ "dist/commonjs/extensions/typescript.d.ts" ], - "extensions/python": [ - "dist/commonjs/extensions/python.d.ts" - ], "extensions/puppeteer": [ "dist/commonjs/extensions/puppeteer.d.ts" ] @@ -70,9 +66,7 @@ }, "dependencies": { "@trigger.dev/core": "workspace:3.3.16", - "@trigger.dev/sdk": "workspace:3.3.16", "pkg-types": "^1.1.3", - "tinyexec": "^0.3.2", "tinyglobby": "^0.2.2", "tsconfck": "3.1.3" }, @@ -156,17 +150,6 @@ "default": "./dist/commonjs/extensions/typescript.js" } }, - "./extensions/python": { - "import": { - "@triggerdotdev/source": "./src/extensions/python.ts", - "types": "./dist/esm/extensions/python.d.ts", - "default": "./dist/esm/extensions/python.js" - }, - "require": { - "types": "./dist/commonjs/extensions/python.d.ts", - "default": "./dist/commonjs/extensions/python.js" - } - }, "./extensions/puppeteer": { "import": { "@triggerdotdev/source": "./src/extensions/puppeteer.ts", diff --git a/packages/python/CHANGELOG.md b/packages/python/CHANGELOG.md new file mode 100644 index 0000000000..abac457942 --- /dev/null +++ b/packages/python/CHANGELOG.md @@ -0,0 +1 @@ +# @trigger.dev/python diff --git a/packages/python/LICENSE b/packages/python/LICENSE new file mode 100644 index 0000000000..b448fb3800 --- /dev/null +++ b/packages/python/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2025 Trigger.dev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/packages/python/README.md b/packages/python/README.md new file mode 100644 index 0000000000..3804bf2f2b --- /dev/null +++ b/packages/python/README.md @@ -0,0 +1,3 @@ +# Official Python extension for Trigger.dev + +View the full documentation [here](https://trigger.dev/docs) diff --git a/packages/python/package.json b/packages/python/package.json new file mode 100644 index 0000000000..7d032b6c8a --- /dev/null +++ b/packages/python/package.json @@ -0,0 +1,93 @@ +{ + "name": "@trigger.dev/python", + "version": "3.3.16", + "description": "Python runtime and build extension for Trigger.dev", + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "repository": { + "type": "git", + "url": "https://github.com/triggerdotdev/trigger.dev", + "directory": "packages/python" + }, + "type": "module", + "files": [ + "dist" + ], + "tshy": { + "selfLink": false, + "main": true, + "module": true, + "project": "./tsconfig.src.json", + "exports": { + "./package.json": "./package.json", + ".": "./src/index.ts", + "./extension": "./src/extension.ts" + }, + "sourceDialects": [ + "@triggerdotdev/source" + ] + }, + "typesVersions": { + "*": { + "extension": [ + "dist/commonjs/extension.d.ts" + ] + } + }, + "scripts": { + "clean": "rimraf dist", + "build": "tshy && pnpm run update-version", + "dev": "tshy --watch", + "typecheck": "tsc --noEmit -p tsconfig.src.json", + "update-version": "tsx ../../scripts/updateVersion.ts", + "check-exports": "attw --pack ." + }, + "dependencies": { + "@trigger.dev/build": "workspace:3.3.16", + "@trigger.dev/core": "workspace:3.3.16", + "@trigger.dev/sdk": "workspace:3.3.16", + "tinyexec": "^0.3.2" + }, + "devDependencies": { + "@types/node": "20.14.14", + "rimraf": "6.0.1", + "tshy": "^3.0.2", + "typescript": "^5.5.4", + "tsx": "4.17.0", + "esbuild": "^0.23.0", + "@arethetypeswrong/cli": "^0.15.4" + }, + "engines": { + "node": ">=18.20.0" + }, + "exports": { + "./package.json": "./package.json", + ".": { + "import": { + "@triggerdotdev/source": "./src/index.ts", + "types": "./dist/esm/index.d.ts", + "default": "./dist/esm/index.js" + }, + "require": { + "types": "./dist/commonjs/index.d.ts", + "default": "./dist/commonjs/index.js" + } + }, + "./extension": { + "import": { + "@triggerdotdev/source": "./src/extension.ts", + "types": "./dist/esm/extension.d.ts", + "default": "./dist/esm/extension.js" + }, + "require": { + "types": "./dist/commonjs/extension.d.ts", + "default": "./dist/commonjs/extension.js" + } + } + }, + "main": "./dist/commonjs/index.js", + "types": "./dist/commonjs/index.d.ts", + "module": "./dist/esm/index.js" +} diff --git a/packages/build/src/extensions/python.ts b/packages/python/src/extension.ts similarity index 64% rename from packages/build/src/extensions/python.ts rename to packages/python/src/extension.ts index 3caa1a4300..ea0c4140fb 100644 --- a/packages/build/src/extensions/python.ts +++ b/packages/python/src/extension.ts @@ -1,10 +1,8 @@ import fs from "node:fs"; import assert from "node:assert"; -import { additionalFiles } from "./core/additionalFiles.js"; +import { additionalFiles } from "@trigger.dev/build/extensions/core"; import { BuildManifest } from "@trigger.dev/core/v3"; import { BuildContext, BuildExtension } from "@trigger.dev/core/v3/build"; -import { logger } from "@trigger.dev/sdk/v3"; -import { x, Options as XOptions, Result } from "tinyexec"; export type PythonOptions = { requirements?: string[]; @@ -118,61 +116,4 @@ class PythonExtension implements BuildExtension { } } -export const run = async ( - scriptArgs: string[] = [], - options: Partial = {} -): Promise => { - const pythonBin = process.env.PYTHON_BIN_PATH || "python"; - - return await logger.trace("Python call", async (span) => { - span.addEvent("Properties", { - command: `${pythonBin} ${scriptArgs.join(" ")}`, - }); - - const result = await x(pythonBin, scriptArgs, { - ...options, - throwOnError: false, // Ensure errors are handled manually - }); - - span.addEvent("Output", { ...result }); - - if (result.exitCode !== 0) { - logger.error(result.stderr, { ...result }); - throw new Error(`Python command exited with non-zero code ${result.exitCode}`); - } - - return result; - }); -}; - -export const runScript = ( - scriptPath: string, - scriptArgs: string[] = [], - options: Partial = {} -) => { - assert(scriptPath, "Script path is required"); - assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); - - return run([scriptPath, ...scriptArgs], options); -}; - -export const runInline = async (scriptContent: string, options: Partial = {}) => { - assert(scriptContent, "Script content is required"); - - const tmpFile = `/tmp/script_${Date.now()}.py`; - await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); - - try { - return await runScript(tmpFile, [], options); - } finally { - try { - await fs.promises.unlink(tmpFile); - } catch (error) { - logger.warn(`Failed to clean up temporary file ${tmpFile}:`, { - error: (error as Error).stack || (error as Error).message, - }); - } - } -}; - -export default { run, runScript, runInline }; +export default pythonExtension; diff --git a/packages/python/src/index.ts b/packages/python/src/index.ts new file mode 100644 index 0000000000..aefcfddbe4 --- /dev/null +++ b/packages/python/src/index.ts @@ -0,0 +1,63 @@ +import fs from "node:fs"; +import assert from "node:assert"; +import { logger } from "@trigger.dev/sdk/v3"; +import { x, Options as XOptions, Result } from "tinyexec"; + +export const run = async ( + scriptArgs: string[] = [], + options: Partial = {} +): Promise => { + const pythonBin = process.env.PYTHON_BIN_PATH || "python"; + + return await logger.trace("Python call", async (span) => { + span.addEvent("Properties", { + command: `${pythonBin} ${scriptArgs.join(" ")}`, + }); + + const result = await x(pythonBin, scriptArgs, { + ...options, + throwOnError: false, // Ensure errors are handled manually + }); + + span.addEvent("Output", { ...result }); + + if (result.exitCode !== 0) { + logger.error(result.stderr, { ...result }); + throw new Error(`Python command exited with non-zero code ${result.exitCode}`); + } + + return result; + }); +}; + +export const runScript = ( + scriptPath: string, + scriptArgs: string[] = [], + options: Partial = {} +) => { + assert(scriptPath, "Script path is required"); + assert(fs.existsSync(scriptPath), `Script does not exist: ${scriptPath}`); + + return run([scriptPath, ...scriptArgs], options); +}; + +export const runInline = async (scriptContent: string, options: Partial = {}) => { + assert(scriptContent, "Script content is required"); + + const tmpFile = `/tmp/script_${Date.now()}.py`; + await fs.promises.writeFile(tmpFile, scriptContent, { mode: 0o600 }); + + try { + return await runScript(tmpFile, [], options); + } finally { + try { + await fs.promises.unlink(tmpFile); + } catch (error) { + logger.warn(`Failed to clean up temporary file ${tmpFile}:`, { + error: (error as Error).stack || (error as Error).message, + }); + } + } +}; + +export default { run, runScript, runInline }; diff --git a/packages/python/tsconfig.json b/packages/python/tsconfig.json new file mode 100644 index 0000000000..16881b51b6 --- /dev/null +++ b/packages/python/tsconfig.json @@ -0,0 +1,8 @@ +{ + "extends": "../../.configs/tsconfig.base.json", + "references": [ + { + "path": "./tsconfig.src.json" + } + ] +} diff --git a/packages/python/tsconfig.src.json b/packages/python/tsconfig.src.json new file mode 100644 index 0000000000..db06c53317 --- /dev/null +++ b/packages/python/tsconfig.src.json @@ -0,0 +1,10 @@ +{ + "extends": "./tsconfig.json", + "include": ["./src/**/*.ts"], + "compilerOptions": { + "isolatedDeclarations": false, + "composite": true, + "sourceMap": true, + "customConditions": ["@triggerdotdev/source"] + } +} diff --git a/references/v3-catalog/trigger.config.ts b/references/v3-catalog/trigger.config.ts index 359260be39..b06685ade8 100644 --- a/references/v3-catalog/trigger.config.ts +++ b/references/v3-catalog/trigger.config.ts @@ -7,7 +7,7 @@ import { ffmpeg, syncEnvVars } from "@trigger.dev/build/extensions/core"; import { puppeteer } from "@trigger.dev/build/extensions/puppeteer"; import { prismaExtension } from "@trigger.dev/build/extensions/prisma"; import { emitDecoratorMetadata } from "@trigger.dev/build/extensions/typescript"; -import { pythonExtension } from "@trigger.dev/build/extensions/python"; +import { pythonExtension } from "@trigger.dev/python/extension"; import { defineConfig } from "@trigger.dev/sdk/v3"; export { handleError } from "./src/handleError.js"; From ec5c3904b8123333f8b77f1db0d7d677750c6c8d Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 25 Feb 2025 06:39:02 +0100 Subject: [PATCH 25/29] fix build package readme --- packages/build/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/build/README.md b/packages/build/README.md index d3ea2f35f9..fdf6403c2f 100644 --- a/packages/build/README.md +++ b/packages/build/README.md @@ -1,3 +1,3 @@ -# Official TypeScript SDK for Trigger.dev +# Official Build Package of Trigger.dev -View the full documentation for the [here](https://trigger.dev/docs) +View the full documentation [here](https://trigger.dev/docs) From cccf3e08de014b11735651fb2a253350c8e3d47b Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 25 Feb 2025 06:39:12 +0100 Subject: [PATCH 26/29] update lock file --- pnpm-lock.yaml | 43 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 37 insertions(+), 6 deletions(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a7419aa5a0..6358b71bfc 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1038,15 +1038,9 @@ importers: '@trigger.dev/core': specifier: workspace:3.3.16 version: link:../core - '@trigger.dev/sdk': - specifier: workspace:3.3.14 - version: link:../trigger-sdk pkg-types: specifier: ^1.1.3 version: 1.1.3 - tinyexec: - specifier: ^0.3.2 - version: 0.3.2 tinyglobby: specifier: ^0.2.2 version: 0.2.2 @@ -1423,6 +1417,43 @@ importers: specifier: ^1.6.0 version: 1.6.0(@types/node@20.14.14) + packages/python: + dependencies: + '@trigger.dev/build': + specifier: workspace:3.3.16 + version: link:../build + '@trigger.dev/core': + specifier: workspace:3.3.16 + version: link:../core + '@trigger.dev/sdk': + specifier: workspace:3.3.16 + version: link:../trigger-sdk + tinyexec: + specifier: ^0.3.2 + version: 0.3.2 + devDependencies: + '@arethetypeswrong/cli': + specifier: ^0.15.4 + version: 0.15.4 + '@types/node': + specifier: 20.14.14 + version: 20.14.14 + esbuild: + specifier: ^0.23.0 + version: 0.23.0 + rimraf: + specifier: 6.0.1 + version: 6.0.1 + tshy: + specifier: ^3.0.2 + version: 3.0.2 + tsx: + specifier: 4.17.0 + version: 4.17.0 + typescript: + specifier: ^5.5.4 + version: 5.5.4 + packages/react-hooks: dependencies: '@trigger.dev/core': From a43306154cb34026b1aa87f7759d073de2e39e3a Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 25 Feb 2025 07:28:50 +0100 Subject: [PATCH 27/29] add documentation to python's package --- packages/python/README.md | 103 +++++++++++++++++++++++++++++++++++++- 1 file changed, 101 insertions(+), 2 deletions(-) diff --git a/packages/python/README.md b/packages/python/README.md index 3804bf2f2b..7dd697b46c 100644 --- a/packages/python/README.md +++ b/packages/python/README.md @@ -1,3 +1,102 @@ -# Official Python extension for Trigger.dev +# Python Extension for Trigger.dev -View the full documentation [here](https://trigger.dev/docs) +The Python extension enhances Trigger.dev's build process by enabling limited support for executing Python scripts within your tasks. + +## Overview + +This extension introduces the pythonExtension build extension, which offers several key capabilities: + +- **Install Python Dependencies (Except in Dev):** Automatically installs Python and specified dependencies using pip. +- **Requirements File Support:** You can specify dependencies in a requirements.txt file. +- **Inline Requirements:** Define dependencies directly within your trigger.config.ts file using the requirements option. +- **Virtual Environment:** Creates a virtual environment (/opt/venv) inside containers to isolate Python dependencies. +- **Helper Functions:** Provides a variety of functions for executing Python code: + - run: Executes Python commands with proper environment setup. + - runInline: Executes inline Python code directly from Node. + - runScript: Executes standalone .py script files. +- **Custom Python Path:** In development, you can configure pythonBinaryPath to point to a custom Python installation. + +## Usage + +1. Add the extension to your trigger.config.ts file: + +```typescript +import { defineConfig } from "@trigger.dev/sdk/v3"; +import pythonExtension from "@trigger.dev/python/extension"; + +export default defineConfig({ + project: "", + build: { + extensions: [ + pythonExtension({ + requirementsFile: "./requirements.txt", // Optional: Path to your requirements file + pythonBinaryPath: path.join(rootDir, `.venv/bin/python`), // Optional: Custom Python binary path + scripts: ["my_script.py"], // List of Python scripts to include + }), + ], + }, +}); +``` + +2. (Optional) Create a requirements.txt file in your project root with the necessary Python dependencies. + +3. Execute Python scripts within your tasks using one of the provided functions: + +### Running a Python Script + +```typescript +import { task } from "@trigger.dev/sdk/v3"; +import python from "@trigger.dev/python"; + +export const myScript = task({ + id: "my-python-script", + run: async () => { + const result = await python.runScript("my_script.py", ["hello", "world"]); + return result.stdout; + }, +}); +``` + +### Running Inline Python Code + +```typescript +import { task } from "@trigger.dev/sdk/v3"; +import python from "@trigger.dev/python"; + +export const myTask = task({ + id: "to_datetime-task", + run: async () => { + const result = await python.runInline(` +import pandas as pd + +pandas.to_datetime("${+new Date() / 1000}") +`); + return result.stdout; + }, +}); +``` + +### Running Lower-Level Commands + +```typescript +import { task } from "@trigger.dev/sdk/v3"; +import python from "@trigger.dev/python"; + +export const pythonVersionTask = task({ + id: "python-version-task", + run: async () => { + const result = await python.run(["--version"]); + return result.stdout; // Expected output: Python 3.12.8 + }, +}); +``` + +## Limitations + +- This is a **partial implementation** and does not provide full Python support as an execution runtime for tasks. +- Only basic Python script execution is supported; scripts are not automatically copied to staging/production containers. +- Manual intervention may be required for installing and configuring binary dependencies in development environments. + +## Additional Information + +For more detailed documentation, visit the official docs at [Trigger.dev Documentation](https://trigger.dev/docs). From 751ac55b2e632a6de226b82ffff12c5aa06ded99 Mon Sep 17 00:00:00 2001 From: zvictor Date: Tue, 25 Feb 2025 20:31:06 +0100 Subject: [PATCH 28/29] add missing dependency --- pnpm-lock.yaml | 3 +++ references/v3-catalog/package.json | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6358b71bfc..0802b03a8b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -1965,6 +1965,9 @@ importers: '@trigger.dev/build': specifier: workspace:* version: link:../../packages/build + '@trigger.dev/python': + specifier: workspace:* + version: link:../../packages/python '@types/email-reply-parser': specifier: ^1.4.2 version: 1.4.2 diff --git a/references/v3-catalog/package.json b/references/v3-catalog/package.json index c9de9cf913..5103a91cea 100644 --- a/references/v3-catalog/package.json +++ b/references/v3-catalog/package.json @@ -73,6 +73,7 @@ "@opentelemetry/sdk-trace-node": "^1.22.0", "@opentelemetry/semantic-conventions": "^1.22.0", "@trigger.dev/build": "workspace:*", + "@trigger.dev/python": "workspace:*", "@types/email-reply-parser": "^1.4.2", "@types/fluent-ffmpeg": "^2.1.26", "@types/node": "20.4.2", @@ -85,4 +86,4 @@ "tsconfig-paths": "^4.2.0", "typescript": "^5.5.4" } -} \ No newline at end of file +} From 7eddd20783a5b70ac670825eb82f24f383368fac Mon Sep 17 00:00:00 2001 From: Eric Allam Date: Wed, 26 Feb 2025 13:34:30 +0000 Subject: [PATCH 29/29] Update little-trains-begin.md --- .changeset/little-trains-begin.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.changeset/little-trains-begin.md b/.changeset/little-trains-begin.md index 15c54cbb0f..11e4b8dc22 100644 --- a/.changeset/little-trains-begin.md +++ b/.changeset/little-trains-begin.md @@ -1,5 +1,5 @@ --- -"@trigger.dev/python": minor +"@trigger.dev/python": patch --- Introduced a new Python extension to enhance the build process. It now allows users to execute Python scripts with improved support and error handling.