pythonExtension and python runtime improvements

Author: ericallam

packages/build/package.json

@@ -23,6 +23,7 @@
     "exports": {
       "./package.json": "./package.json",
       ".": "./src/index.ts",
+      "./internal": "./src/internal.ts",
       "./extensions": "./src/extensions/index.ts",
       "./extensions/core": "./src/extensions/core.ts",
       "./extensions/prisma": "./src/extensions/prisma.ts",
@@ -36,6 +37,9 @@
   },
   "typesVersions": {
     "*": {
+      "internal": [
+        "dist/commonjs/internal.d.ts"
+      ],
       "extensions": [
         "dist/commonjs/extensions/index.d.ts"
       ],
@@ -95,6 +99,17 @@
         "default": "./dist/commonjs/index.js"
       }
     },
+    "./internal": {
+      "import": {
+        "@triggerdotdev/source": "./src/internal.ts",
+        "types": "./dist/esm/internal.d.ts",
+        "default": "./dist/esm/internal.js"
+      },
+      "require": {
+        "types": "./dist/commonjs/internal.d.ts",
+        "default": "./dist/commonjs/internal.js"
+      }
+    },
     "./extensions": {
       "import": {
         "@triggerdotdev/source": "./src/extensions/index.ts",

packages/build/src/extensions/core/additionalFiles.ts

@@ -1,7 +1,5 @@
-import { relative, join, posix, dirname } from "node:path";
-import { glob } from "tinyglobby";
-import { copyFile, mkdir } from "node:fs/promises";
 import { BuildExtension } from "@trigger.dev/core/v3/build";
+import { addAdditionalFilesToBuild } from "../../internal/additionalFiles.js";
 
 export type AdditionalFilesOptions = {
   files: string[];
@@ -11,86 +9,7 @@ export function additionalFiles(options: AdditionalFilesOptions): BuildExtension
   return {
     name: "additionalFiles",
     async onBuildComplete(context, manifest) {
-      // Copy any static assets to the destination
-      const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, {
-        cwd: context.workingDir,
-      });
-
-      for (const { assets, matcher } of staticAssets) {
-        if (assets.length === 0) {
-          console.warn("No files found for matcher", matcher);
-        }
-      }
-
-      await copyStaticAssets(staticAssets);
+      await addAdditionalFilesToBuild("additionalFiles", options, context, manifest);
     },
   };
 }
-
-type MatchedStaticAssets = { source: string; destination: string }[];
-
-type FoundStaticAssetFiles = Array<{
-  matcher: string;
-  assets: MatchedStaticAssets;
-}>;
-
-async function findStaticAssetFiles(
-  matchers: string[],
-  destinationPath: string,
-  options?: { cwd?: string; ignore?: string[] }
-): Promise<FoundStaticAssetFiles> {
-  const result: FoundStaticAssetFiles = [];
-
-  for (const matcher of matchers) {
-    const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options);
-
-    result.push({ matcher, assets });
-  }
-
-  return result;
-}
-
-async function findStaticAssetsForMatcher(
-  matcher: string,
-  destinationPath: string,
-  options?: { cwd?: string; ignore?: string[] }
-): Promise<MatchedStaticAssets> {
-  const result: MatchedStaticAssets = [];
-
-  const files = await glob({
-    patterns: [matcher],
-    cwd: options?.cwd,
-    ignore: options?.ignore ?? [],
-    onlyFiles: true,
-    absolute: true,
-  });
-
-  let matches = 0;
-
-  for (const file of files) {
-    matches++;
-
-    const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file)
-      .split(posix.sep)
-      .filter((p) => p !== "..")
-      .join(posix.sep);
-
-    const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir);
-
-    result.push({
-      source: file,
-      destination: relativeDestinationPath,
-    });
-  }
-
-  return result;
-}
-
-async function copyStaticAssets(staticAssetFiles: FoundStaticAssetFiles): Promise<void> {
-  for (const { assets } of staticAssetFiles) {
-    for (const { source, destination } of assets) {
-      await mkdir(dirname(destination), { recursive: true });
-      await copyFile(source, destination);
-    }
-  }
-}

packages/build/src/internal.ts

@@ -0,0 +1 @@
+export * from "./internal/additionalFiles.js";

packages/build/src/internal/additionalFiles.ts

@@ -0,0 +1,106 @@
+import { BuildManifest } from "@trigger.dev/core/v3";
+import { BuildContext } from "@trigger.dev/core/v3/build";
+import { copyFile, mkdir } from "node:fs/promises";
+import { dirname, join, posix, relative } from "node:path";
+import { glob } from "tinyglobby";
+
+export type AdditionalFilesOptions = {
+  files: string[];
+};
+
+export async function addAdditionalFilesToBuild(
+  source: string,
+  options: AdditionalFilesOptions,
+  context: BuildContext,
+  manifest: BuildManifest
+) {
+  // Copy any static assets to the destination
+  const staticAssets = await findStaticAssetFiles(options.files ?? [], manifest.outputPath, {
+    cwd: context.workingDir,
+  });
+
+  for (const { assets, matcher } of staticAssets) {
+    if (assets.length === 0) {
+      context.logger.warn(`[${source}] No files found for matcher`, matcher);
+    } else {
+      context.logger.debug(`[${source}] Found ${assets.length} files for matcher`, matcher);
+    }
+  }
+
+  await copyStaticAssets(staticAssets, source, context);
+}
+
+type MatchedStaticAssets = { source: string; destination: string }[];
+
+type FoundStaticAssetFiles = Array<{
+  matcher: string;
+  assets: MatchedStaticAssets;
+}>;
+
+async function findStaticAssetFiles(
+  matchers: string[],
+  destinationPath: string,
+  options?: { cwd?: string; ignore?: string[] }
+): Promise<FoundStaticAssetFiles> {
+  const result: FoundStaticAssetFiles = [];
+
+  for (const matcher of matchers) {
+    const assets = await findStaticAssetsForMatcher(matcher, destinationPath, options);
+
+    result.push({ matcher, assets });
+  }
+
+  return result;
+}
+
+async function findStaticAssetsForMatcher(
+  matcher: string,
+  destinationPath: string,
+  options?: { cwd?: string; ignore?: string[] }
+): Promise<MatchedStaticAssets> {
+  const result: MatchedStaticAssets = [];
+
+  const files = await glob({
+    patterns: [matcher],
+    cwd: options?.cwd,
+    ignore: options?.ignore ?? [],
+    onlyFiles: true,
+    absolute: true,
+  });
+
+  let matches = 0;
+
+  for (const file of files) {
+    matches++;
+
+    const pathInsideDestinationDir = relative(options?.cwd ?? process.cwd(), file)
+      .split(posix.sep)
+      .filter((p) => p !== "..")
+      .join(posix.sep);
+
+    const relativeDestinationPath = join(destinationPath, pathInsideDestinationDir);
+
+    result.push({
+      source: file,
+      destination: relativeDestinationPath,
+    });
+  }
+
+  return result;
+}
+
+async function copyStaticAssets(
+  staticAssetFiles: FoundStaticAssetFiles,
+  sourceName: string,
+  context: BuildContext
+): Promise<void> {
+  for (const { assets } of staticAssetFiles) {
+    for (const { source, destination } of assets) {
+      await mkdir(dirname(destination), { recursive: true });
+
+      context.logger.debug(`[${sourceName}] Copying ${source} to ${destination}`);
+
+      await copyFile(source, destination);
+    }
+  }
+}

packages/core/src/v3/logger/index.ts

@@ -1,6 +1,6 @@
 import { NoopTaskLogger, TaskLogger } from "./taskLogger.js";
 import { getGlobal, registerGlobal, unregisterGlobal } from "../utils/globals.js";
-import { Span } from "@opentelemetry/api";
+import { Span, SpanOptions } from "@opentelemetry/api";
 
 const API_NAME = "logger";
 
@@ -47,8 +47,8 @@ export class LoggerAPI implements TaskLogger {
     this.#getTaskLogger().error(message, metadata);
   }
 
-  public trace<T>(name: string, fn: (span: Span) => Promise<T>): Promise<T> {
-    return this.#getTaskLogger().trace(name, fn);
+  public trace<T>(name: string, fn: (span: Span) => Promise<T>, options?: SpanOptions): Promise<T> {
+    return this.#getTaskLogger().trace(name, fn, options);
   }
 
   #getTaskLogger(): TaskLogger {

packages/python/README.md

@@ -14,24 +14,24 @@ This extension introduces the <code>pythonExtension</code> build extension, whic
   - <code>run</code>: Executes Python commands with proper environment setup.
   - <code>runInline</code>: Executes inline Python code directly from Node.
   - <code>runScript</code>: Executes standalone <code>.py</code> script files.
-- **Custom Python Path:** In development, you can configure <code>pythonBinaryPath</code> to point to a custom Python installation.
+- **Custom Python Path:** In development, you can configure <code>devPythonBinaryPath</code> to point to a custom Python installation.
 
 ## Usage
 
 1. Add the extension to your <code>trigger.config.ts</code> file:
 
 ```typescript
 import { defineConfig } from "@trigger.dev/sdk/v3";
-import pythonExtension from "@trigger.dev/python/extension";
+import { pythonExtension } from "@trigger.dev/python/extension";
 
 export default defineConfig({
   project: "<project ref>",
   build: {
     extensions: [
       pythonExtension({
         requirementsFile: "./requirements.txt", // Optional: Path to your requirements file
-        pythonBinaryPath: path.join(rootDir, `.venv/bin/python`), // Optional: Custom Python binary path
-        scripts: ["my_script.py"], // List of Python scripts to include
+        devPythonBinaryPath: ".venv/bin/python", // Optional: Custom Python binary path
+        scripts: ["src/python/**/*.py"], // Glob pattern for Python scripts
       }),
     ],
   },
@@ -40,13 +40,34 @@ export default defineConfig({
 
 2. (Optional) Create a <code>requirements.txt</code> file in your project root with the necessary Python dependencies.
 
+```plaintext title="requirements.txt"
+pandas==1.3.3
+numpy==1.21.2
+```
+
+```typescript title="trigger.config.ts"
+import { defineConfig } from "@trigger.dev/sdk/v3";
+import { pythonExtension } from "@trigger.dev/python/extension";
+
+export default defineConfig({
+  project: "<project ref>",
+  build: {
+    extensions: [
+      pythonExtension({
+        requirementsFile: "./requirements.txt",
+      }),
+    ],
+  },
+});
+```
+
 3. Execute Python scripts within your tasks using one of the provided functions:
 
 ### Running a Python Script
 
 ```typescript
 import { task } from "@trigger.dev/sdk/v3";
-import python from "@trigger.dev/python";
+import { python } from "@trigger.dev/python";
 
 export const myScript = task({
   id: "my-python-script",
@@ -61,15 +82,15 @@ export const myScript = task({
 
 ```typescript
 import { task } from "@trigger.dev/sdk/v3";
-import python from "@trigger.dev/python";
+import { python } from "@trigger.dev/python";
 
 export const myTask = task({
   id: "to_datetime-task",
   run: async () => {
     const result = await python.runInline(`
 import pandas as pd
 
-pandas.to_datetime("${+new Date() / 1000}")
+pd.to_datetime("${+new Date() / 1000}")
 `);
     return result.stdout;
   },
@@ -80,7 +101,7 @@ pandas.to_datetime("${+new Date() / 1000}")
 
 ```typescript
 import { task } from "@trigger.dev/sdk/v3";
-import python from "@trigger.dev/python";
+import { python } from "@trigger.dev/python";
 
 export const pythonVersionTask = task({
   id: "python-version-task",
@@ -94,7 +115,6 @@ export const pythonVersionTask = task({
 ## Limitations
 
 - This is a **partial implementation** and does not provide full Python support as an execution runtime for tasks.
-- Only basic Python script execution is supported; scripts are not automatically copied to staging/production containers.
 - Manual intervention may be required for installing and configuring binary dependencies in development environments.
 
 ## Additional Information

packages/python/package.json

@@ -45,9 +45,7 @@
     "check-exports": "attw --pack ."
   },
   "dependencies": {
-    "@trigger.dev/build": "workspace:3.3.16",
     "@trigger.dev/core": "workspace:3.3.16",
-    "@trigger.dev/sdk": "workspace:3.3.16",
     "tinyexec": "^0.3.2"
   },
   "devDependencies": {
@@ -57,7 +55,13 @@
     "typescript": "^5.5.4",
     "tsx": "4.17.0",
     "esbuild": "^0.23.0",
-    "@arethetypeswrong/cli": "^0.15.4"
+    "@arethetypeswrong/cli": "^0.15.4",
+    "@trigger.dev/build": "workspace:3.3.16",
+    "@trigger.dev/sdk": "workspace:3.3.16"
+  },
+  "peerDependencies": {
+    "@trigger.dev/sdk": "workspace:^3.3.16",
+    "@trigger.dev/build": "workspace:^3.3.16"
   },
   "engines": {
     "node": ">=18.20.0"