Skip to content

Commit 85725dc

Browse files
nirganirga
authored
fix(langchain): report input/output (#131)
1 parent a861a36 commit 85725dc

File tree

7 files changed

+167
-22
lines changed

7 files changed

+167
-22
lines changed

‎package-lock.json‎

Lines changed: 2 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎packages/instrumentation-azure/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
"@opentelemetry/core": "^1.22.0",
4040
"@opentelemetry/instrumentation": "^0.49.0",
4141
"@opentelemetry/semantic-conventions": "^1.22.0",
42-
"@traceloop/ai-semantic-conventions": "^0.5.6"
42+
"@traceloop/ai-semantic-conventions": "*"
4343
},
4444
"devDependencies": {
4545
"@azure/openai": "^1.0.0-beta.10",

‎packages/instrumentation-langchain/package.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@
3939
"@opentelemetry/core": "^1.22.0",
4040
"@opentelemetry/instrumentation": "^0.49.0",
4141
"@opentelemetry/semantic-conventions": "^1.22.0",
42-
"@traceloop/ai-semantic-conventions": "^0.0.16"
42+
"@traceloop/ai-semantic-conventions": "*"
4343
},
4444
"devDependencies": {
4545
"@langchain/community": "^0.0.34",

‎packages/instrumentation-langchain/src/instrumentation.ts‎

Lines changed: 34 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -12,18 +12,22 @@
1212
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and
1313
* limitations under the License.
1414
*/
15+
import { context } from "@opentelemetry/api";
1516
import {
1617
InstrumentationBase,
1718
InstrumentationModuleDefinition,
1819
InstrumentationNodeModuleDefinition,
1920
} from "@opentelemetry/instrumentation";
21+
import { CONTEXT_KEY_ALLOW_TRACE_CONTENT } from "@traceloop/ai-semantic-conventions";
2022
import { LangChainInstrumentationConfig } from "./types";
2123
import { taskWrapper, workflowWrapper } from "./utils";
2224
import type * as ChainsModule from "langchain/chains";
2325
import type * as AgentsModule from "langchain/agents";
2426
import type * as ToolsModule from "langchain/tools";
2527

2628
export class LangChainInstrumentation extends InstrumentationBase<any> {
29+
protected override _config!: LangChainInstrumentationConfig;
30+
2731
constructor(config: LangChainInstrumentationConfig = {}) {
2832
super("@traceloop/instrumentation-langchain", "0.3.0", config);
2933
}
@@ -82,12 +86,16 @@ export class LangChainInstrumentation extends InstrumentationBase<any> {
8286
this._wrap(
8387
moduleExports.RetrievalQAChain.prototype,
8488
"_call",
85-
workflowWrapper(this.tracer, "retrieval_qa.workflow"),
89+
workflowWrapper(
90+
this.tracer,
91+
this._shouldSendPrompts(),
92+
"retrieval_qa.workflow",
93+
),
8694
);
8795
this._wrap(
8896
moduleExports.BaseChain.prototype,
8997
"call",
90-
taskWrapper(this.tracer),
98+
taskWrapper(this.tracer, this._shouldSendPrompts()),
9199
);
92100
return moduleExports;
93101
}
@@ -104,7 +112,11 @@ export class LangChainInstrumentation extends InstrumentationBase<any> {
104112
this._wrap(
105113
moduleExports.AgentExecutor.prototype,
106114
"_call",
107-
workflowWrapper(this.tracer, "langchain.agent"),
115+
workflowWrapper(
116+
this.tracer,
117+
this._shouldSendPrompts(),
118+
"langchain.agent",
119+
),
108120
);
109121
return moduleExports;
110122
}
@@ -118,7 +130,11 @@ export class LangChainInstrumentation extends InstrumentationBase<any> {
118130

119131
moduleExports.openLLMetryPatched = true;
120132

121-
this._wrap(moduleExports.Tool.prototype, "call", taskWrapper(this.tracer));
133+
this._wrap(
134+
moduleExports.Tool.prototype,
135+
"call",
136+
taskWrapper(this.tracer, this._shouldSendPrompts()),
137+
);
122138
return moduleExports;
123139
}
124140

@@ -149,4 +165,18 @@ export class LangChainInstrumentation extends InstrumentationBase<any> {
149165
this._unwrap(moduleExports.AgentExecutor.prototype, "_call");
150166
return moduleExports;
151167
}
168+
169+
private _shouldSendPrompts() {
170+
const contextShouldSendPrompts = context
171+
.active()
172+
.getValue(CONTEXT_KEY_ALLOW_TRACE_CONTENT);
173+
174+
if (contextShouldSendPrompts !== undefined) {
175+
return !!contextShouldSendPrompts;
176+
}
177+
178+
return this._config.traceContent !== undefined
179+
? this._config.traceContent
180+
: true;
181+
}
152182
}

‎packages/instrumentation-langchain/src/types.ts‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
11
import { InstrumentationConfig } from "@opentelemetry/instrumentation";
22

3-
export type LangChainInstrumentationConfig = InstrumentationConfig;
3+
export interface LangChainInstrumentationConfig extends InstrumentationConfig {
4+
/**
5+
* Whether to log prompts, completions and embeddings on traces.
6+
* @default true
7+
*/
8+
traceContent?: boolean;
9+
}

‎packages/instrumentation-langchain/src/utils.ts‎

Lines changed: 70 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import {
77

88
export function genericWrapper(
99
tracer: Tracer,
10+
shouldSendPrompts: boolean,
1011
spanKind: TraceloopSpanKindValues,
1112
spanName?: string,
1213
) {
@@ -17,6 +18,34 @@ export function genericWrapper(
1718
spanName || `langchain.${spanKind}.${this.constructor.name}`,
1819
);
1920
span.setAttribute(SpanAttributes.TRACELOOP_SPAN_KIND, spanKind);
21+
22+
if (shouldSendPrompts) {
23+
try {
24+
if (
25+
args.length === 1 &&
26+
typeof args[0] === "object" &&
27+
!(args[0] instanceof Map)
28+
) {
29+
span.setAttribute(
30+
SpanAttributes.TRACELOOP_ENTITY_INPUT,
31+
JSON.stringify({ args: [], kwargs: args[0] }),
32+
);
33+
} else {
34+
span.setAttribute(
35+
SpanAttributes.TRACELOOP_ENTITY_INPUT,
36+
JSON.stringify({
37+
args: args.map((arg) =>
38+
arg instanceof Map ? Array.from(arg.entries()) : arg,
39+
),
40+
kwargs: {},
41+
}),
42+
);
43+
}
44+
} catch {
45+
/* empty */
46+
}
47+
}
48+
2049
const execContext = trace.setSpan(context.active(), span);
2150
const execPromise = safeExecuteInTheMiddle(
2251
() => {
@@ -31,8 +60,25 @@ export function genericWrapper(
3160
.then((result: any) => {
3261
return new Promise((resolve) => {
3362
span.setStatus({ code: SpanStatusCode.OK });
34-
span.end();
35-
resolve(result);
63+
64+
try {
65+
if (shouldSendPrompts) {
66+
if (result instanceof Map) {
67+
span.setAttribute(
68+
SpanAttributes.TRACELOOP_ENTITY_OUTPUT,
69+
JSON.stringify(Array.from(result.entries())),
70+
);
71+
} else {
72+
span.setAttribute(
73+
SpanAttributes.TRACELOOP_ENTITY_OUTPUT,
74+
JSON.stringify(result),
75+
);
76+
}
77+
}
78+
} finally {
79+
span.end();
80+
resolve(result);
81+
}
3682
});
3783
})
3884
.catch((error: Error) => {
@@ -50,10 +96,28 @@ export function genericWrapper(
5096
};
5197
}
5298

53-
export function taskWrapper(tracer: Tracer, spanName?: string) {
54-
return genericWrapper(tracer, TraceloopSpanKindValues.TASK, spanName);
99+
export function taskWrapper(
100+
tracer: Tracer,
101+
shouldSendPrompts: boolean,
102+
spanName?: string,
103+
) {
104+
return genericWrapper(
105+
tracer,
106+
shouldSendPrompts,
107+
TraceloopSpanKindValues.TASK,
108+
spanName,
109+
);
55110
}
56111

57-
export function workflowWrapper(tracer: Tracer, spanName: string) {
58-
return genericWrapper(tracer, TraceloopSpanKindValues.WORKFLOW, spanName);
112+
export function workflowWrapper(
113+
tracer: Tracer,
114+
shouldSendPrompts: boolean,
115+
spanName: string,
116+
) {
117+
return genericWrapper(
118+
tracer,
119+
shouldSendPrompts,
120+
TraceloopSpanKindValues.WORKFLOW,
121+
spanName,
122+
);
59123
}

‎packages/instrumentation-langchain/test/instrumentation.test.ts‎

Lines changed: 52 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,17 @@ describe("Test Langchain instrumentation", async function () {
141141
assert.ok(result);
142142
assert.ok(agentSpan);
143143
assert.strictEqual(agentSpan.attributes["traceloop.span.kind"], "workflow");
144+
assert.ok(agentSpan.attributes["traceloop.entity.input"]);
145+
assert.ok(agentSpan.attributes["traceloop.entity.output"]);
146+
assert.strictEqual(
147+
JSON.parse(agentSpan.attributes["traceloop.entity.input"].toString())
148+
.args[0].input,
149+
"Solve `5 * (10 + 2)`",
150+
);
151+
assert.deepEqual(
152+
JSON.parse(agentSpan.attributes["traceloop.entity.output"].toString()),
153+
result,
154+
);
144155
}).timeout(60000);
145156

146157
it("should set attributes in span for chain instrumentation", async () => {
@@ -207,6 +218,20 @@ describe("Test Langchain instrumentation", async function () {
207218
retrievalQAChainSpan.attributes["traceloop.span.kind"],
208219
"task",
209220
);
221+
assert.ok(retrievalQAChainSpan.attributes["traceloop.entity.input"]);
222+
assert.ok(retrievalQAChainSpan.attributes["traceloop.entity.output"]);
223+
assert.strictEqual(
224+
JSON.parse(
225+
retrievalQAChainSpan.attributes["traceloop.entity.input"].toString(),
226+
).kwargs.query,
227+
"What did the author do growing up?",
228+
);
229+
assert.deepEqual(
230+
JSON.parse(
231+
retrievalQAChainSpan.attributes["traceloop.entity.output"].toString(),
232+
),
233+
answer,
234+
);
210235
}).timeout(300000);
211236

212237
it("should set attributes in span for retrieval qa instrumentation", async () => {
@@ -253,6 +278,20 @@ describe("Test Langchain instrumentation", async function () {
253278
retrievalQASpan.attributes["traceloop.span.kind"],
254279
"workflow",
255280
);
281+
assert.ok(retrievalQASpan.attributes["traceloop.entity.input"]);
282+
assert.ok(retrievalQASpan.attributes["traceloop.entity.output"]);
283+
assert.strictEqual(
284+
JSON.parse(
285+
retrievalQASpan.attributes["traceloop.entity.input"].toString(),
286+
).args[0].query,
287+
"What did the president say about Justice Breyer?",
288+
);
289+
assert.deepEqual(
290+
JSON.parse(
291+
retrievalQASpan.attributes["traceloop.entity.output"].toString(),
292+
),
293+
answer,
294+
);
256295
}).timeout(300000);
257296

258297
it("should set correct attributes in span for LCEL", async () => {
@@ -285,5 +324,18 @@ describe("Test Langchain instrumentation", async function () {
285324
assert.ok(result);
286325
assert.ok(wikipediaSpan);
287326
assert.strictEqual(wikipediaSpan.attributes["traceloop.span.kind"], "task");
327+
assert.ok(wikipediaSpan.attributes["traceloop.entity.input"]);
328+
assert.ok(wikipediaSpan.attributes["traceloop.entity.output"]);
329+
assert.strictEqual(
330+
JSON.parse(wikipediaSpan.attributes["traceloop.entity.input"].toString())
331+
.args[0],
332+
"Current prime minister of Malaysia",
333+
);
334+
assert.deepEqual(
335+
JSON.parse(
336+
wikipediaSpan.attributes["traceloop.entity.output"].toString(),
337+
),
338+
result,
339+
);
288340
}).timeout(300000);
289341
});

0 commit comments

Comments
 (0)