Skip to content

Commit 719c6f8

Browse files
godwhoagodwhoa
authored
fix(anthropic): add system prompt to span attributes (#612)
1 parent 82ef7d7 commit 719c6f8

File tree

3 files changed

+259
-5
lines changed

3 files changed

+259
-5
lines changed

‎packages/instrumentation-anthropic/recordings/Test-Anthropic-instrumentation_3769946143/should-place-system-prompt-first-for-messages_507270015/recording.har

Lines changed: 210 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,210 @@
1+
{
2+
"log": {
3+
"_recordingName": "Test Anthropic instrumentation/should place system prompt first for messages",
4+
"creator": {
5+
"comment": "persister:fs",
6+
"name": "Polly.JS",
7+
"version": "6.0.6"
8+
},
9+
"entries": [
10+
{
11+
"_id": "d46eba4b0503410ea2ca82210094482a",
12+
"_order": 0,
13+
"cache": {},
14+
"request": {
15+
"bodySize": 172,
16+
"cookies": [],
17+
"headers": [
18+
{
19+
"name": "accept",
20+
"value": "application/json"
21+
},
22+
{
23+
"name": "anthropic-version",
24+
"value": "2023-06-01"
25+
},
26+
{
27+
"name": "content-type",
28+
"value": "application/json"
29+
},
30+
{
31+
"name": "user-agent",
32+
"value": "Anthropic/JS 0.50.4"
33+
},
34+
{
35+
"name": "x-stainless-arch",
36+
"value": "arm64"
37+
},
38+
{
39+
"name": "x-stainless-lang",
40+
"value": "js"
41+
},
42+
{
43+
"name": "x-stainless-os",
44+
"value": "MacOS"
45+
},
46+
{
47+
"name": "x-stainless-package-version",
48+
"value": "0.50.4"
49+
},
50+
{
51+
"name": "x-stainless-retry-count",
52+
"value": "0"
53+
},
54+
{
55+
"name": "x-stainless-runtime",
56+
"value": "node"
57+
},
58+
{
59+
"name": "x-stainless-runtime-version",
60+
"value": "v22.13.0"
61+
},
62+
{
63+
"name": "x-stainless-timeout",
64+
"value": "600"
65+
}
66+
],
67+
"headersSize": 525,
68+
"httpVersion": "HTTP/1.1",
69+
"method": "POST",
70+
"postData": {
71+
"mimeType": "application/json",
72+
"params": [],
73+
"text": "{\"max_tokens\":10,\"model\":\"claude-3-opus-20240229\",\"system\":\"You are a helpful assistant\",\"messages\":[{\"role\":\"user\",\"content\":\"Hi\"},{\"role\":\"assistant\",\"content\":\"Hello\"}]}"
74+
},
75+
"queryString": [],
76+
"url": "https://api.anthropic.com/v1/messages"
77+
},
78+
"response": {
79+
"bodySize": 354,
80+
"content": {
81+
"mimeType": "application/json",
82+
"size": 354,
83+
"text": "{\"id\":\"msg_01U3xjyNSAcrYd1yog1ADg24\",\"type\":\"message\",\"role\":\"assistant\",\"model\":\"claude-3-opus-20240229\",\"content\":[{\"type\":\"text\",\"text\":\"! How can I assist you today?\"}],\"stop_reason\":\"max_tokens\",\"stop_sequence\":null,\"usage\":{\"input_tokens\":14,\"cache_creation_input_tokens\":0,\"cache_read_input_tokens\":0,\"output_tokens\":10,\"service_tier\":\"standard\"}}"
84+
},
85+
"cookies": [],
86+
"headers": [
87+
{
88+
"name": "anthropic-organization-id",
89+
"value": "2964ad9e-7e53-457c-8ade-3305b3d8bf4a"
90+
},
91+
{
92+
"name": "anthropic-ratelimit-input-tokens-limit",
93+
"value": "40000"
94+
},
95+
{
96+
"name": "anthropic-ratelimit-input-tokens-remaining",
97+
"value": "40000"
98+
},
99+
{
100+
"name": "anthropic-ratelimit-input-tokens-reset",
101+
"value": "2025-07-16T18:27:59Z"
102+
},
103+
{
104+
"name": "anthropic-ratelimit-output-tokens-limit",
105+
"value": "8000"
106+
},
107+
{
108+
"name": "anthropic-ratelimit-output-tokens-remaining",
109+
"value": "8000"
110+
},
111+
{
112+
"name": "anthropic-ratelimit-output-tokens-reset",
113+
"value": "2025-07-16T18:28:00Z"
114+
},
115+
{
116+
"name": "anthropic-ratelimit-requests-limit",
117+
"value": "1000"
118+
},
119+
{
120+
"name": "anthropic-ratelimit-requests-remaining",
121+
"value": "999"
122+
},
123+
{
124+
"name": "anthropic-ratelimit-requests-reset",
125+
"value": "2025-07-16T18:27:59Z"
126+
},
127+
{
128+
"name": "anthropic-ratelimit-tokens-limit",
129+
"value": "48000"
130+
},
131+
{
132+
"name": "anthropic-ratelimit-tokens-remaining",
133+
"value": "48000"
134+
},
135+
{
136+
"name": "anthropic-ratelimit-tokens-reset",
137+
"value": "2025-07-16T18:27:59Z"
138+
},
139+
{
140+
"name": "cf-cache-status",
141+
"value": "DYNAMIC"
142+
},
143+
{
144+
"name": "cf-ray",
145+
"value": "96038e8a1aba2e99-HYD"
146+
},
147+
{
148+
"name": "connection",
149+
"value": "keep-alive"
150+
},
151+
{
152+
"name": "content-encoding",
153+
"value": "gzip"
154+
},
155+
{
156+
"name": "content-type",
157+
"value": "application/json"
158+
},
159+
{
160+
"name": "date",
161+
"value": "Wed, 16 Jul 2025 18:28:01 GMT"
162+
},
163+
{
164+
"name": "request-id",
165+
"value": "req_011CRBAquydCMPrCJEUky2Sx"
166+
},
167+
{
168+
"name": "server",
169+
"value": "cloudflare"
170+
},
171+
{
172+
"name": "strict-transport-security",
173+
"value": "max-age=31536000; includeSubDomains; preload"
174+
},
175+
{
176+
"name": "transfer-encoding",
177+
"value": "chunked"
178+
},
179+
{
180+
"name": "via",
181+
"value": "1.1 google"
182+
},
183+
{
184+
"name": "x-robots-tag",
185+
"value": "none"
186+
}
187+
],
188+
"headersSize": 1048,
189+
"httpVersion": "HTTP/1.1",
190+
"redirectURL": "",
191+
"status": 200,
192+
"statusText": "OK"
193+
},
194+
"startedDateTime": "2025-07-16T18:27:59.601Z",
195+
"time": 1429,
196+
"timings": {
197+
"blocked": -1,
198+
"connect": -1,
199+
"dns": -1,
200+
"receive": 0,
201+
"send": 0,
202+
"ssl": -1,
203+
"wait": 1429
204+
}
205+
}
206+
],
207+
"pages": [],
208+
"version": "1.2"
209+
}
210+
}

‎packages/instrumentation-anthropic/src/instrumentation.ts

Lines changed: 20 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -220,15 +220,30 @@ export class AnthropicInstrumentation extends InstrumentationBase {
220220

221221
if (this._shouldSendPrompts()) {
222222
if (type === "chat") {
223+
let promptIndex = 0;
224+
225+
// If a system prompt is provided, it should always be first
226+
if ("system" in params && params.system !== undefined) {
227+
attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`] = "system";
228+
attributes[`${SpanAttributes.LLM_PROMPTS}.0.content`] =
229+
typeof params.system === "string"
230+
? params.system
231+
: JSON.stringify(params.system);
232+
promptIndex += 1;
233+
}
234+
223235
params.messages.forEach((message, index) => {
224-
attributes[`${SpanAttributes.LLM_PROMPTS}.${index}.role`] =
236+
const currentIndex = index + promptIndex;
237+
attributes[`${SpanAttributes.LLM_PROMPTS}.${currentIndex}.role`] =
225238
message.role;
226239
if (typeof message.content === "string") {
227-
attributes[`${SpanAttributes.LLM_PROMPTS}.${index}.content`] =
228-
(message.content as string) || "";
240+
attributes[
241+
`${SpanAttributes.LLM_PROMPTS}.${currentIndex}.content`
242+
] = (message.content as string) || "";
229243
} else {
230-
attributes[`${SpanAttributes.LLM_PROMPTS}.${index}.content`] =
231-
JSON.stringify(message.content);
244+
attributes[
245+
`${SpanAttributes.LLM_PROMPTS}.${currentIndex}.content`
246+
] = JSON.stringify(message.content);
232247
}
233248
});
234249
} else {

‎packages/instrumentation-anthropic/test/instrumentation.test.ts

Lines changed: 29 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -297,4 +297,33 @@ describe("Test Anthropic instrumentation", async function () {
297297
chatSpan.attributes[`${SpanAttributes.LLM_USAGE_TOTAL_TOKENS}`],
298298
);
299299
}).timeout(30000);
300+
301+
it("should place system prompt first for messages", async () => {
302+
const msg = await anthropic.messages.create({
303+
max_tokens: 10,
304+
model: "claude-3-opus-20240229",
305+
system: "You are a helpful assistant",
306+
messages: [
307+
{ role: "user", content: "Hi" },
308+
{ role: "assistant", content: "Hello" },
309+
],
310+
});
311+
312+
assert.ok(msg);
313+
const span = memoryExporter.getFinishedSpans().at(-1);
314+
315+
assert.ok(span);
316+
assert.strictEqual(
317+
span.attributes[`${SpanAttributes.LLM_PROMPTS}.0.role`],
318+
"system",
319+
);
320+
assert.strictEqual(
321+
span.attributes[`${SpanAttributes.LLM_PROMPTS}.0.content`],
322+
"You are a helpful assistant",
323+
);
324+
assert.strictEqual(
325+
span.attributes[`${SpanAttributes.LLM_PROMPTS}.1.role`],
326+
"user",
327+
);
328+
}).timeout(30000);
300329
});

0 commit comments

Comments
 (0)