WIP langsmith & AI SDK integration

Author: ericallam

packages/cli-v3/src/entryPoints/dev-run-worker.ts

@@ -129,6 +129,7 @@ async function bootstrap() {
   const tracingSDK = new TracingSDK({
     url: env.OTEL_EXPORTER_OTLP_ENDPOINT ?? "http://0.0.0.0:4318",
     instrumentations: config.instrumentations ?? [],
+    exporters: config.exporters ?? [],
     diagLogLevel: (env.OTEL_LOG_LEVEL as TracingDiagnosticLogLevel) ?? "none",
     forceFlushTimeoutMillis: 30_000,
   });

packages/core/src/v3/config.ts

@@ -56,6 +56,8 @@ export type TriggerConfig = {
    */
   instrumentations?: Array<Instrumentation>;
 
+  exporters?: Array<any>;
+
   /**
    * Specify a custom path to your tsconfig file. This is useful if you have a custom tsconfig file that you want to use.
    */

packages/core/src/v3/otel/tracingSDK.ts

@@ -85,6 +85,7 @@ export type TracingSDKConfig = {
   forceFlushTimeoutMillis?: number;
   resource?: IResource;
   instrumentations?: Instrumentation[];
+  exporters?: SpanExporter[];
   diagLogLevel?: TracingDiagnosticLogLevel;
 };
 
@@ -153,6 +154,10 @@ export class TracingSDK {
       )
     );
 
+    for (const exporter of config.exporters ?? []) {
+      traceProvider.addSpanProcessor(new SimpleSpanProcessor(exporter));
+    }
+
     traceProvider.register();
 
     registerInstrumentations({

pnpm-lock.yaml

@@ -27,6 +27,7 @@
     "class-variance-authority": "^0.7.0",
     "clsx": "^2.1.1",
     "date-fns": "^4.1.0",
+    "langsmith": "^0.2.15",
     "lucide-react": "^0.451.0",
     "next": "14.2.15",
     "openai": "^4.68.4",

references/nextjs-realtime/package.json

@@ -4,6 +4,7 @@ import { streamText, type TextStreamPart } from "ai";
 import { setTimeout } from "node:timers/promises";
 import { z } from "zod";
 import OpenAI from "openai";
+import { AISDKExporter } from "langsmith/vercel";
 
 const openaiSDK = new OpenAI({
   apiKey: process.env.OPENAI_API_KEY,
@@ -98,13 +99,14 @@ export const openaiStreaming = schemaTask({
   run: async ({ model, prompt }) => {
     logger.info("Running OpenAI model", { model, prompt });
 
+    const telemetrySettings = AISDKExporter.getSettings();
+
+    logger.info("Telemetry settings", { telemetrySettings });
+
     const result = streamText({
       model: openai(model),
       prompt,
-      tools: {
-        getWeather: weatherTask.tool,
-      },
-      maxSteps: 10,
+      experimental_telemetry: telemetrySettings,
     });
 
     const stream = await metadata.stream("openai", result.fullStream);

references/nextjs-realtime/src/trigger/ai.ts

@@ -1,9 +1,59 @@
 import { defineConfig } from "@trigger.dev/sdk/v3";
 import { rscExtension } from "@trigger.dev/rsc";
+import { AISDKExporter } from "langsmith/vercel";
+import { Client } from "langsmith";
+
+class LangsmithSpanExporterWrapper {
+  constructor(
+    private underlyingExporter: any,
+    private transformSpan: (span: any) => any | undefined = (span) => {
+      if (span.attributes["$span.partial"]) {
+        // Skip partial spans
+        return;
+      }
+
+      // Check if this is an attempt span
+      if (span.name.startsWith("Attempt ")) {
+        // Create a new span that wraps the original but modifies spanContext
+        const spanContext = span.spanContext();
+
+        return {
+          ...span,
+          spanContext: () => spanContext,
+          parentSpanId: undefined,
+        };
+      }
+      return span;
+    }
+  ) {}
+
+  export(spans: any[], resultCallback: (result: any) => void): void {
+    const modifiedSpans = spans.map(this.transformSpan);
+    this.underlyingExporter.export(modifiedSpans.filter(Boolean), resultCallback);
+  }
+
+  shutdown(): Promise<void> {
+    return this.underlyingExporter.shutdown();
+  }
+
+  forceFlush?(): Promise<void> {
+    return this.underlyingExporter.forceFlush
+      ? this.underlyingExporter.forceFlush()
+      : Promise.resolve();
+  }
+}
+
+const client = new Client();
+
+const exporter = new AISDKExporter({
+  debug: true,
+  client,
+});
 
 export default defineConfig({
   project: "proj_bzhdaqhlymtuhlrcgbqy",
   dirs: ["./src/trigger"],
+  exporters: [new LangsmithSpanExporterWrapper(exporter)],
   build: {
     extensions: [rscExtension({ reactDomEnvironment: "worker" })],
   },